diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 13:11:00 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 13:11:00 +0200 |
| commit | 37cbaef9ceb828c667da8f398027ba5415ee7475 (patch) | |
| tree | 6103e92a9030766e78a35df00f7a1155d0d0ecf5 /libffwd/model.c | |
| parent | cf7d8169ec12cb3adf90d4ff78b3257e74985302 (diff) | |
| download | langdet-37cbaef9ceb828c667da8f398027ba5415ee7475.tar.gz langdet-37cbaef9ceb828c667da8f398027ba5415ee7475.zip | |
Rename CPU library API and sources to langdet
Diffstat (limited to 'libffwd/model.c')
| -rw-r--r-- | libffwd/model.c | 159 |
1 files changed, 0 insertions, 159 deletions
diff --git a/libffwd/model.c b/libffwd/model.c deleted file mode 100644 index 81073f3..0000000 --- a/libffwd/model.c +++ /dev/null @@ -1,159 +0,0 @@ -#include "model_internal.h" -#include "config.h" - -#include <stdio.h> -#include <stdlib.h> - -int tensor_has_supported_shape(const safetensors_file_t *sf, - const safetensor_t *t, - const char *name, - const int64_t *shape, - int ndim) { - if (t->dtype != DTYPE_F32) { - fprintf(stderr, "ffwd: expected F32 tensor: %s\n", name); - return 0; - } - if (t->ndim != ndim) { - fprintf(stderr, "ffwd: bad rank for %s: got %d, expected %d\n", name, t->ndim, ndim); - return 0; - } - - size_t numel = 1; - for (int i = 0; i < ndim; i++) { - if (t->shape[i] != shape[i]) { - fprintf(stderr, - "ffwd: bad shape for %s at dim %d: got %lld, expected %lld\n", - name, i, (long long)t->shape[i], (long long)shape[i]); - return 0; - } - if (shape[i] <= 0 || (uint64_t)shape[i] > SIZE_MAX / numel) { - fprintf(stderr, "ffwd: shape too large or empty for %s\n", name); - return 0; - } - numel *= (size_t)shape[i]; - } - size_t bytes; - if (mul_size(numel, sizeof(float), &bytes) != 0 || t->data_size != bytes) { - fprintf(stderr, "ffwd: invalid data size for %s\n", name); - return 0; - } - size_t data_start = 8 + sf->header_size; - if (data_start > sf->file_size || t->data_offset > sf->file_size - data_start || - t->data_size > sf->file_size - data_start - t->data_offset) { - fprintf(stderr, "ffwd: tensor data out of bounds for %s\n", name); - return 0; - } - return 1; -} - -static const float *load_weight(multi_safetensors_t *ms, const char *name, - const int64_t *shape, int ndim) { - safetensors_file_t *sf = NULL; - const safetensor_t *t = multi_safetensors_find(ms, name, &sf); - if (!t) { - fprintf(stderr, "ffwd: tensor not found: %s\n", name); - return NULL; - } - if (!tensor_has_supported_shape(sf, t, name, shape, ndim)) - return NULL; - return safetensors_data(sf, t); -} - -static int load_bert_weights(ffwd_model_t *model, multi_safetensors_t *ms) { - const ffwd_config_t *cfg = &model->config; - ffwd_weights_t *w = &model->weights; - char name[256]; - int hidden = cfg->hidden_size, inter = cfg->intermediate_size; - - w->embed_tokens = load_weight(ms, "roberta.embeddings.word_embeddings.weight", - (int64_t[]){cfg->vocab_size, hidden}, 2); - w->position_embeddings = load_weight(ms, "roberta.embeddings.position_embeddings.weight", - (int64_t[]){cfg->max_position_embeddings, hidden}, 2); - w->token_type_embeddings = load_weight(ms, "roberta.embeddings.token_type_embeddings.weight", - (int64_t[]){cfg->type_vocab_size, hidden}, 2); - w->ffwd_ln_w = load_weight(ms, "roberta.embeddings.LayerNorm.weight", (int64_t[]){hidden}, 1); - w->ffwd_ln_b = load_weight(ms, "roberta.embeddings.LayerNorm.bias", (int64_t[]){hidden}, 1); - if (!w->embed_tokens || !w->position_embeddings || !w->token_type_embeddings || - !w->ffwd_ln_w || !w->ffwd_ln_b) - return -1; - - w->layers = (ffwd_layer_t *)calloc(cfg->n_layers, sizeof(ffwd_layer_t)); - if (!w->layers) - return -1; - - for (int i = 0; i < cfg->n_layers; i++) { - ffwd_layer_t *l = &w->layers[i]; - -#define BERT_W(field, suffix, d0, d1) \ - do { \ - const int64_t e[2] = {(d0), (d1)}; \ - snprintf(name, sizeof(name), "roberta.encoder.layer.%d." suffix, i); \ - l->field = load_weight(ms, name, e, 2); \ - if (!l->field) \ - return -1; \ - } while (0) -#define BERT_V(field, suffix, d0) \ - do { \ - const int64_t e[1] = {(d0)}; \ - snprintf(name, sizeof(name), "roberta.encoder.layer.%d." suffix, i); \ - l->field = load_weight(ms, name, e, 1); \ - if (!l->field) \ - return -1; \ - } while (0) - - BERT_W(wq, "attention.self.query.weight", hidden, hidden); - BERT_V(q_bias, "attention.self.query.bias", hidden); - BERT_W(wk, "attention.self.key.weight", hidden, hidden); - BERT_V(k_bias, "attention.self.key.bias", hidden); - BERT_W(wv, "attention.self.value.weight", hidden, hidden); - BERT_V(v_bias, "attention.self.value.bias", hidden); - BERT_W(wo, "attention.output.dense.weight", hidden, hidden); - BERT_V(o_bias, "attention.output.dense.bias", hidden); - BERT_V(input_norm, "attention.output.LayerNorm.weight", hidden); - BERT_V(attn_ln_bias, "attention.output.LayerNorm.bias", hidden); - BERT_W(up_proj, "intermediate.dense.weight", inter, hidden); - BERT_V(ffn_inter_bias, "intermediate.dense.bias", inter); - BERT_W(down_proj, "output.dense.weight", hidden, inter); - BERT_V(ffn_out_bias, "output.dense.bias", hidden); - BERT_V(post_attn_norm, "output.LayerNorm.weight", hidden); - BERT_V(ffn_ln_bias, "output.LayerNorm.bias", hidden); - -#undef BERT_W -#undef BERT_V - } - return 0; -} - -ffwd_model_t *ffwd_model_load(const char *model_dir) { - if (!model_dir) - return NULL; - ffwd_model_t *model = (ffwd_model_t *)calloc(1, sizeof(ffwd_model_t)); - if (!model) - return NULL; - if (ffwd_config_parse(&model->config, model_dir) != 0) - goto fail; - model->safetensors = multi_safetensors_open(model_dir); - if (!model->safetensors) { - fprintf(stderr, "ffwd_model_load: failed to open safetensors in %s\n", model_dir); - goto fail; - } - if (load_bert_weights(model, model->safetensors) != 0) - goto fail; - return model; - -fail: - ffwd_model_free(model); - return NULL; -} - -void ffwd_model_free(ffwd_model_t *model) { - if (!model) - return; - multi_safetensors_close(model->safetensors); - free(model->weights.layers); - free(model); -} - -const ffwd_config_t *ffwd_model_config(const ffwd_model_t *model) { - return model ? &model->config : NULL; -} |
