From 64c3a1b528a88b4ae21bf3cbcff2537200477806 Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Sun, 13 Sep 2026 20:03:39 +0200 Subject: Init --- src/weights.c | 189 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 src/weights.c (limited to 'src/weights.c') diff --git a/src/weights.c b/src/weights.c new file mode 100644 index 0000000..42e3c9c --- /dev/null +++ b/src/weights.c @@ -0,0 +1,189 @@ +/* Weight blob reader. + * + * The file is a header, a fixed-size table of named tensors, then the tensor + * data at 64-byte boundaries. Parsing checks that every tensor the engine + * needs is present with the exact byte count that shape implies, so a blob + * exported from a different checkpoint or a different layout is rejected at + * load time rather than producing plausible wrong numbers. */ + +#include "weights.h" + +#include +#include +#include +#include +#include +#include + +#include + +#define MAGIC "PSG1" +#define VERSION 1u +#define NAME_LEN 24 + +typedef struct { + char magic[4]; + uint32_t version; + uint32_t count; + uint32_t total; +} blob_header; + +typedef struct { + char name[NAME_LEN]; + uint32_t offset; + uint32_t nbytes; + uint32_t digest; +} blob_entry; + +typedef struct { + const unsigned char *base; + size_t len; + const blob_entry *table; + uint32_t count; + int verify; + int failed; +} reader; + +static uint32_t fnv1a(const unsigned char *p, size_t n) { + uint32_t h = 0x811C9DC5u; + for (size_t i = 0; i < n; ++i) + h = (h ^ p[i]) * 0x01000193u; + return h; +} + +/* Looks up one tensor and checks it is exactly floats long. A miss sets the + * reader's failure flag rather than returning it, so the caller can list every + * tensor once and test the flag at the end. */ +static const float *take(reader *r, const char *name, size_t floats) { + if (r->failed) + return NULL; + for (uint32_t i = 0; i < r->count; ++i) { + const blob_entry *e = &r->table[i]; + if (strncmp(e->name, name, NAME_LEN) != 0) + continue; + if (e->nbytes != floats * sizeof(float) || e->offset % 64 != 0 || e->offset > r->len || + e->nbytes > r->len - e->offset) + break; + const unsigned char *data = r->base + e->offset; + if (r->verify && fnv1a(data, e->nbytes) != e->digest) + break; + return (const float *)(const void *)data; + } + r->failed = 1; + return NULL; +} + +int pyseg_weights_parse(const void *blob, size_t len, int verify, pyseg_weights *out) { + if (!blob || len < sizeof(blob_header)) + return -1; + const blob_header *h = (const blob_header *)blob; + if (memcmp(h->magic, MAGIC, 4) != 0 || h->version != VERSION) + return -1; + if (h->count > 1024) + return -1; + const size_t table_bytes = (size_t)h->count * sizeof(blob_entry); + if (len < sizeof(blob_header) + table_bytes || len < h->total) + return -1; + + reader r = { + .base = (const unsigned char *)blob, + .len = len, + .table = (const blob_entry *)(const void *)((const unsigned char *)blob + sizeof *h), + .count = h->count, + .verify = verify, + .failed = 0, + }; + + memset(out, 0, sizeof *out); + out->wavnorm = take(&r, "wavnorm", 2); + out->sinc_w = take(&r, "sinc_w", 80 * 256); + + static const int norm_ch[3] = {80, 60, 60}; + for (int i = 0; i < 3; ++i) { + char name[NAME_LEN]; + snprintf(name, sizeof name, "norm%d_w", i + 1); + out->norm_w[i] = take(&r, name, (size_t)norm_ch[i]); + snprintf(name, sizeof name, "norm%d_b", i + 1); + out->norm_b[i] = take(&r, name, (size_t)norm_ch[i]); + } + + static const int conv_k[2] = {400, 300}; + for (int i = 0; i < 2; ++i) { + char name[NAME_LEN]; + snprintf(name, sizeof name, "conv%d_w", i + 2); + out->conv_w[i] = take(&r, name, 60u * (size_t)conv_k[i]); + snprintf(name, sizeof name, "conv%d_b", i + 2); + out->conv_b[i] = take(&r, name, 60); + } + + for (int l = 0; l < 4; ++l) { + const size_t in_dim = l == 0 ? 60 : 256; + for (int d = 0; d < 2; ++d) { + char name[NAME_LEN]; + const char tag = d == 0 ? 'f' : 'b'; + snprintf(name, sizeof name, "lstm%d%c_ih", l, tag); + out->lstm[l][d].ih = take(&r, name, 512 * in_dim); + snprintf(name, sizeof name, "lstm%d%c_hh", l, tag); + out->lstm[l][d].hh = take(&r, name, 512 * 128); + snprintf(name, sizeof name, "lstm%d%c_bias", l, tag); + out->lstm[l][d].bias = take(&r, name, 512); + } + } + + static const int lin_k[2] = {256, 128}; + for (int i = 0; i < 2; ++i) { + char name[NAME_LEN]; + snprintf(name, sizeof name, "lin%d_w", i); + out->lin_w[i] = take(&r, name, 128u * (size_t)lin_k[i]); + snprintf(name, sizeof name, "lin%d_b", i); + out->lin_b[i] = take(&r, name, 128); + } + + out->cls_w = take(&r, "cls_w", 7 * 128); + out->cls_b = take(&r, "cls_b", 7); + + return r.failed ? -1 : 0; +} + +int pyseg_weights_map(const char *path, int verify, pyseg_weights *out, void **map, size_t *map_len) { + const int fd = open(path, O_RDONLY); + if (fd < 0) + return -1; + struct stat st; + if (fstat(fd, &st) != 0 || st.st_size <= 0) { + close(fd); + return -1; + } + const size_t len = (size_t)st.st_size; + void *addr = mmap(NULL, len, PROT_READ, MAP_PRIVATE, fd, 0); + close(fd); + if (addr == MAP_FAILED) + return -1; + if (pyseg_weights_parse(addr, len, verify, out) != 0) { + munmap(addr, len); + return -1; + } + *map = addr; + *map_len = len; + return 0; +} + +void pyseg_weights_unmap(void *map, size_t len) { + if (map) + munmap(map, len); +} + +#ifdef PYSEG_EMBED_WEIGHTS +extern const unsigned char pyseg_weights_blob[]; +extern const size_t pyseg_weights_blob_len; + +const void *pyseg_weights_embedded(size_t *len) { + *len = pyseg_weights_blob_len; + return pyseg_weights_blob; +} +#else +const void *pyseg_weights_embedded(size_t *len) { + *len = 0; + return NULL; +} +#endif -- cgit v1.3.1