aboutsummaryrefslogtreecommitdiff
path: root/liblangdet/model.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-09-29 13:11:00 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-09-29 13:11:00 +0200
commit37cbaef9ceb828c667da8f398027ba5415ee7475 (patch)
tree6103e92a9030766e78a35df00f7a1155d0d0ecf5 /liblangdet/model.c
parentcf7d8169ec12cb3adf90d4ff78b3257e74985302 (diff)
downloadlangdet-37cbaef9ceb828c667da8f398027ba5415ee7475.tar.gz
langdet-37cbaef9ceb828c667da8f398027ba5415ee7475.zip
Rename CPU library API and sources to langdet
Diffstat (limited to 'liblangdet/model.c')
-rw-r--r--liblangdet/model.c159
1 files changed, 159 insertions, 0 deletions
diff --git a/liblangdet/model.c b/liblangdet/model.c
new file mode 100644
index 0000000..146bbe7
--- /dev/null
+++ b/liblangdet/model.c
@@ -0,0 +1,159 @@
+#include "model_internal.h"
+#include "config.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+
+int tensor_has_supported_shape(const safetensors_file_t *sf,
+ const safetensor_t *t,
+ const char *name,
+ const int64_t *shape,
+ int ndim) {
+ if (t->dtype != DTYPE_F32) {
+ fprintf(stderr, "langdet: expected F32 tensor: %s\n", name);
+ return 0;
+ }
+ if (t->ndim != ndim) {
+ fprintf(stderr, "langdet: bad rank for %s: got %d, expected %d\n", name, t->ndim, ndim);
+ return 0;
+ }
+
+ size_t numel = 1;
+ for (int i = 0; i < ndim; i++) {
+ if (t->shape[i] != shape[i]) {
+ fprintf(stderr,
+ "langdet: bad shape for %s at dim %d: got %lld, expected %lld\n",
+ name, i, (long long)t->shape[i], (long long)shape[i]);
+ return 0;
+ }
+ if (shape[i] <= 0 || (uint64_t)shape[i] > SIZE_MAX / numel) {
+ fprintf(stderr, "langdet: shape too large or empty for %s\n", name);
+ return 0;
+ }
+ numel *= (size_t)shape[i];
+ }
+ size_t bytes;
+ if (mul_size(numel, sizeof(float), &bytes) != 0 || t->data_size != bytes) {
+ fprintf(stderr, "langdet: invalid data size for %s\n", name);
+ return 0;
+ }
+ size_t data_start = 8 + sf->header_size;
+ if (data_start > sf->file_size || t->data_offset > sf->file_size - data_start ||
+ t->data_size > sf->file_size - data_start - t->data_offset) {
+ fprintf(stderr, "langdet: tensor data out of bounds for %s\n", name);
+ return 0;
+ }
+ return 1;
+}
+
+static const float *load_weight(multi_safetensors_t *ms, const char *name,
+ const int64_t *shape, int ndim) {
+ safetensors_file_t *sf = NULL;
+ const safetensor_t *t = multi_safetensors_find(ms, name, &sf);
+ if (!t) {
+ fprintf(stderr, "langdet: tensor not found: %s\n", name);
+ return NULL;
+ }
+ if (!tensor_has_supported_shape(sf, t, name, shape, ndim))
+ return NULL;
+ return safetensors_data(sf, t);
+}
+
+static int load_bert_weights(langdet_model_t *model, multi_safetensors_t *ms) {
+ const langdet_config_t *cfg = &model->config;
+ langdet_weights_t *w = &model->weights;
+ char name[256];
+ int hidden = cfg->hidden_size, inter = cfg->intermediate_size;
+
+ w->embed_tokens = load_weight(ms, "roberta.embeddings.word_embeddings.weight",
+ (int64_t[]){cfg->vocab_size, hidden}, 2);
+ w->position_embeddings = load_weight(ms, "roberta.embeddings.position_embeddings.weight",
+ (int64_t[]){cfg->max_position_embeddings, hidden}, 2);
+ w->token_type_embeddings = load_weight(ms, "roberta.embeddings.token_type_embeddings.weight",
+ (int64_t[]){cfg->type_vocab_size, hidden}, 2);
+ w->langdet_ln_w = load_weight(ms, "roberta.embeddings.LayerNorm.weight", (int64_t[]){hidden}, 1);
+ w->langdet_ln_b = load_weight(ms, "roberta.embeddings.LayerNorm.bias", (int64_t[]){hidden}, 1);
+ if (!w->embed_tokens || !w->position_embeddings || !w->token_type_embeddings ||
+ !w->langdet_ln_w || !w->langdet_ln_b)
+ return -1;
+
+ w->layers = (langdet_layer_t *)calloc(cfg->n_layers, sizeof(langdet_layer_t));
+ if (!w->layers)
+ return -1;
+
+ for (int i = 0; i < cfg->n_layers; i++) {
+ langdet_layer_t *l = &w->layers[i];
+
+#define BERT_W(field, suffix, d0, d1) \
+ do { \
+ const int64_t e[2] = {(d0), (d1)}; \
+ snprintf(name, sizeof(name), "roberta.encoder.layer.%d." suffix, i); \
+ l->field = load_weight(ms, name, e, 2); \
+ if (!l->field) \
+ return -1; \
+ } while (0)
+#define BERT_V(field, suffix, d0) \
+ do { \
+ const int64_t e[1] = {(d0)}; \
+ snprintf(name, sizeof(name), "roberta.encoder.layer.%d." suffix, i); \
+ l->field = load_weight(ms, name, e, 1); \
+ if (!l->field) \
+ return -1; \
+ } while (0)
+
+ BERT_W(wq, "attention.self.query.weight", hidden, hidden);
+ BERT_V(q_bias, "attention.self.query.bias", hidden);
+ BERT_W(wk, "attention.self.key.weight", hidden, hidden);
+ BERT_V(k_bias, "attention.self.key.bias", hidden);
+ BERT_W(wv, "attention.self.value.weight", hidden, hidden);
+ BERT_V(v_bias, "attention.self.value.bias", hidden);
+ BERT_W(wo, "attention.output.dense.weight", hidden, hidden);
+ BERT_V(o_bias, "attention.output.dense.bias", hidden);
+ BERT_V(input_norm, "attention.output.LayerNorm.weight", hidden);
+ BERT_V(attn_ln_bias, "attention.output.LayerNorm.bias", hidden);
+ BERT_W(up_proj, "intermediate.dense.weight", inter, hidden);
+ BERT_V(ffn_inter_bias, "intermediate.dense.bias", inter);
+ BERT_W(down_proj, "output.dense.weight", hidden, inter);
+ BERT_V(ffn_out_bias, "output.dense.bias", hidden);
+ BERT_V(post_attn_norm, "output.LayerNorm.weight", hidden);
+ BERT_V(ffn_ln_bias, "output.LayerNorm.bias", hidden);
+
+#undef BERT_W
+#undef BERT_V
+ }
+ return 0;
+}
+
+langdet_model_t *langdet_model_load(const char *model_dir) {
+ if (!model_dir)
+ return NULL;
+ langdet_model_t *model = (langdet_model_t *)calloc(1, sizeof(langdet_model_t));
+ if (!model)
+ return NULL;
+ if (langdet_config_parse(&model->config, model_dir) != 0)
+ goto fail;
+ model->safetensors = multi_safetensors_open(model_dir);
+ if (!model->safetensors) {
+ fprintf(stderr, "langdet_model_load: failed to open safetensors in %s\n", model_dir);
+ goto fail;
+ }
+ if (load_bert_weights(model, model->safetensors) != 0)
+ goto fail;
+ return model;
+
+fail:
+ langdet_model_free(model);
+ return NULL;
+}
+
+void langdet_model_free(langdet_model_t *model) {
+ if (!model)
+ return;
+ multi_safetensors_close(model->safetensors);
+ free(model->weights.layers);
+ free(model);
+}
+
+const langdet_config_t *langdet_model_config(const langdet_model_t *model) {
+ return model ? &model->config : NULL;
+}