aboutsummaryrefslogtreecommitdiff
path: root/liblangdet/model.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-09-29 14:38:02 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-09-29 14:38:02 +0200
commit91c6aa07597462705a7f30ee5b97e2c7943b5616 (patch)
treeb16f686279a1f1b0691a6c0d65a544084fcc1c1e /liblangdet/model.c
parent6f98fc8cf3f7cc4b9d523d16492a07d3e2aea8c9 (diff)
downloadlangdet-91c6aa07597462705a7f30ee5b97e2c7943b5616.tar.gz
langdet-91c6aa07597462705a7f30ee5b97e2c7943b5616.zip
Remove unused checkpoint, tokenizer, and attention compatibility paths
Diffstat (limited to 'liblangdet/model.c')
-rw-r--r--liblangdet/model.c91
1 files changed, 47 insertions, 44 deletions
diff --git a/liblangdet/model.c b/liblangdet/model.c
index 146bbe7..53f3692 100644
--- a/liblangdet/model.c
+++ b/liblangdet/model.c
@@ -46,10 +46,9 @@ int tensor_has_supported_shape(const safetensors_file_t *sf,
return 1;
}
-static const float *load_weight(multi_safetensors_t *ms, const char *name,
+static const float *load_weight(const safetensors_file_t *sf, const char *name,
const int64_t *shape, int ndim) {
- safetensors_file_t *sf = NULL;
- const safetensor_t *t = multi_safetensors_find(ms, name, &sf);
+ const safetensor_t *t = safetensors_find(sf, name);
if (!t) {
fprintf(stderr, "langdet: tensor not found: %s\n", name);
return NULL;
@@ -59,20 +58,20 @@ static const float *load_weight(multi_safetensors_t *ms, const char *name,
return safetensors_data(sf, t);
}
-static int load_bert_weights(langdet_model_t *model, multi_safetensors_t *ms) {
+static int load_xlm_roberta_weights(langdet_model_t *model, const safetensors_file_t *sf) {
const langdet_config_t *cfg = &model->config;
langdet_weights_t *w = &model->weights;
char name[256];
int hidden = cfg->hidden_size, inter = cfg->intermediate_size;
- w->embed_tokens = load_weight(ms, "roberta.embeddings.word_embeddings.weight",
+ w->embed_tokens = load_weight(sf, "roberta.embeddings.word_embeddings.weight",
(int64_t[]){cfg->vocab_size, hidden}, 2);
- w->position_embeddings = load_weight(ms, "roberta.embeddings.position_embeddings.weight",
+ w->position_embeddings = load_weight(sf, "roberta.embeddings.position_embeddings.weight",
(int64_t[]){cfg->max_position_embeddings, hidden}, 2);
- w->token_type_embeddings = load_weight(ms, "roberta.embeddings.token_type_embeddings.weight",
+ w->token_type_embeddings = load_weight(sf, "roberta.embeddings.token_type_embeddings.weight",
(int64_t[]){cfg->type_vocab_size, hidden}, 2);
- w->langdet_ln_w = load_weight(ms, "roberta.embeddings.LayerNorm.weight", (int64_t[]){hidden}, 1);
- w->langdet_ln_b = load_weight(ms, "roberta.embeddings.LayerNorm.bias", (int64_t[]){hidden}, 1);
+ w->langdet_ln_w = load_weight(sf, "roberta.embeddings.LayerNorm.weight", (int64_t[]){hidden}, 1);
+ w->langdet_ln_b = load_weight(sf, "roberta.embeddings.LayerNorm.bias", (int64_t[]){hidden}, 1);
if (!w->embed_tokens || !w->position_embeddings || !w->token_type_embeddings ||
!w->langdet_ln_w || !w->langdet_ln_b)
return -1;
@@ -84,42 +83,42 @@ static int load_bert_weights(langdet_model_t *model, multi_safetensors_t *ms) {
for (int i = 0; i < cfg->n_layers; i++) {
langdet_layer_t *l = &w->layers[i];
-#define BERT_W(field, suffix, d0, d1) \
- do { \
- const int64_t e[2] = {(d0), (d1)}; \
- snprintf(name, sizeof(name), "roberta.encoder.layer.%d." suffix, i); \
- l->field = load_weight(ms, name, e, 2); \
- if (!l->field) \
- return -1; \
+#define LOAD_MATRIX(field, suffix, d0, d1) \
+ do { \
+ const int64_t e[2] = {(d0), (d1)}; \
+ snprintf(name, sizeof(name), "roberta.encoder.layer.%d." suffix, i); \
+ l->field = load_weight(sf, name, e, 2); \
+ if (!l->field) \
+ return -1; \
} while (0)
-#define BERT_V(field, suffix, d0) \
- do { \
- const int64_t e[1] = {(d0)}; \
- snprintf(name, sizeof(name), "roberta.encoder.layer.%d." suffix, i); \
- l->field = load_weight(ms, name, e, 1); \
- if (!l->field) \
- return -1; \
+#define LOAD_VECTOR(field, suffix, d0) \
+ do { \
+ const int64_t e[1] = {(d0)}; \
+ snprintf(name, sizeof(name), "roberta.encoder.layer.%d." suffix, i); \
+ l->field = load_weight(sf, name, e, 1); \
+ if (!l->field) \
+ return -1; \
} while (0)
- BERT_W(wq, "attention.self.query.weight", hidden, hidden);
- BERT_V(q_bias, "attention.self.query.bias", hidden);
- BERT_W(wk, "attention.self.key.weight", hidden, hidden);
- BERT_V(k_bias, "attention.self.key.bias", hidden);
- BERT_W(wv, "attention.self.value.weight", hidden, hidden);
- BERT_V(v_bias, "attention.self.value.bias", hidden);
- BERT_W(wo, "attention.output.dense.weight", hidden, hidden);
- BERT_V(o_bias, "attention.output.dense.bias", hidden);
- BERT_V(input_norm, "attention.output.LayerNorm.weight", hidden);
- BERT_V(attn_ln_bias, "attention.output.LayerNorm.bias", hidden);
- BERT_W(up_proj, "intermediate.dense.weight", inter, hidden);
- BERT_V(ffn_inter_bias, "intermediate.dense.bias", inter);
- BERT_W(down_proj, "output.dense.weight", hidden, inter);
- BERT_V(ffn_out_bias, "output.dense.bias", hidden);
- BERT_V(post_attn_norm, "output.LayerNorm.weight", hidden);
- BERT_V(ffn_ln_bias, "output.LayerNorm.bias", hidden);
+ LOAD_MATRIX(wq, "attention.self.query.weight", hidden, hidden);
+ LOAD_VECTOR(q_bias, "attention.self.query.bias", hidden);
+ LOAD_MATRIX(wk, "attention.self.key.weight", hidden, hidden);
+ LOAD_VECTOR(k_bias, "attention.self.key.bias", hidden);
+ LOAD_MATRIX(wv, "attention.self.value.weight", hidden, hidden);
+ LOAD_VECTOR(v_bias, "attention.self.value.bias", hidden);
+ LOAD_MATRIX(wo, "attention.output.dense.weight", hidden, hidden);
+ LOAD_VECTOR(o_bias, "attention.output.dense.bias", hidden);
+ LOAD_VECTOR(input_norm, "attention.output.LayerNorm.weight", hidden);
+ LOAD_VECTOR(attn_ln_bias, "attention.output.LayerNorm.bias", hidden);
+ LOAD_MATRIX(up_proj, "intermediate.dense.weight", inter, hidden);
+ LOAD_VECTOR(ffn_inter_bias, "intermediate.dense.bias", inter);
+ LOAD_MATRIX(down_proj, "output.dense.weight", hidden, inter);
+ LOAD_VECTOR(ffn_out_bias, "output.dense.bias", hidden);
+ LOAD_VECTOR(post_attn_norm, "output.LayerNorm.weight", hidden);
+ LOAD_VECTOR(ffn_ln_bias, "output.LayerNorm.bias", hidden);
-#undef BERT_W
-#undef BERT_V
+#undef LOAD_MATRIX
+#undef LOAD_VECTOR
}
return 0;
}
@@ -132,12 +131,16 @@ langdet_model_t *langdet_model_load(const char *model_dir) {
return NULL;
if (langdet_config_parse(&model->config, model_dir) != 0)
goto fail;
- model->safetensors = multi_safetensors_open(model_dir);
+ char path[1024];
+ int length = snprintf(path, sizeof(path), "%s/model.safetensors", model_dir);
+ if (length < 0 || (size_t)length >= sizeof(path))
+ goto fail;
+ model->safetensors = safetensors_open(path);
if (!model->safetensors) {
fprintf(stderr, "langdet_model_load: failed to open safetensors in %s\n", model_dir);
goto fail;
}
- if (load_bert_weights(model, model->safetensors) != 0)
+ if (load_xlm_roberta_weights(model, model->safetensors) != 0)
goto fail;
return model;
@@ -149,7 +152,7 @@ fail:
void langdet_model_free(langdet_model_t *model) {
if (!model)
return;
- multi_safetensors_close(model->safetensors);
+ safetensors_close(model->safetensors);
free(model->weights.layers);
free(model);
}