#include "model_internal.h" #include "config.h" #include #include int tensor_has_supported_shape(const safetensors_file_t *sf, const safetensor_t *t, const char *name, const int64_t *shape, int ndim) { if (t->dtype != DTYPE_F32) { fprintf(stderr, "langdet: expected F32 tensor: %s\n", name); return 0; } if (t->ndim != ndim) { fprintf(stderr, "langdet: bad rank for %s: got %d, expected %d\n", name, t->ndim, ndim); return 0; } size_t numel = 1; for (int i = 0; i < ndim; i++) { if (t->shape[i] != shape[i]) { fprintf(stderr, "langdet: bad shape for %s at dim %d: got %lld, expected %lld\n", name, i, (long long)t->shape[i], (long long)shape[i]); return 0; } if (shape[i] <= 0 || (uint64_t)shape[i] > SIZE_MAX / numel) { fprintf(stderr, "langdet: shape too large or empty for %s\n", name); return 0; } numel *= (size_t)shape[i]; } size_t bytes; if (mul_size(numel, sizeof(float), &bytes) != 0 || t->data_size != bytes) { fprintf(stderr, "langdet: invalid data size for %s\n", name); return 0; } size_t data_start = 8 + sf->header_size; if (data_start > sf->file_size || t->data_offset > sf->file_size - data_start || t->data_size > sf->file_size - data_start - t->data_offset) { fprintf(stderr, "langdet: tensor data out of bounds for %s\n", name); return 0; } return 1; } static const float *load_weight(const safetensors_file_t *sf, const char *name, const int64_t *shape, int ndim) { const safetensor_t *t = safetensors_find(sf, name); if (!t) { fprintf(stderr, "langdet: tensor not found: %s\n", name); return NULL; } if (!tensor_has_supported_shape(sf, t, name, shape, ndim)) return NULL; return safetensors_data(sf, t); } static int load_xlm_roberta_weights(langdet_model_t *model, const safetensors_file_t *sf) { const langdet_config_t *cfg = &model->config; langdet_weights_t *w = &model->weights; char name[256]; int hidden = cfg->hidden_size, inter = cfg->intermediate_size; w->embed_tokens = load_weight(sf, "roberta.embeddings.word_embeddings.weight", (int64_t[]){cfg->vocab_size, hidden}, 2); w->position_embeddings = load_weight(sf, "roberta.embeddings.position_embeddings.weight", (int64_t[]){cfg->max_position_embeddings, hidden}, 2); w->token_type_embeddings = load_weight(sf, "roberta.embeddings.token_type_embeddings.weight", (int64_t[]){cfg->type_vocab_size, hidden}, 2); w->langdet_ln_w = load_weight(sf, "roberta.embeddings.LayerNorm.weight", (int64_t[]){hidden}, 1); w->langdet_ln_b = load_weight(sf, "roberta.embeddings.LayerNorm.bias", (int64_t[]){hidden}, 1); if (!w->embed_tokens || !w->position_embeddings || !w->token_type_embeddings || !w->langdet_ln_w || !w->langdet_ln_b) return -1; w->layers = (langdet_layer_t *)calloc(cfg->n_layers, sizeof(langdet_layer_t)); if (!w->layers) return -1; for (int i = 0; i < cfg->n_layers; i++) { langdet_layer_t *l = &w->layers[i]; #define LOAD_MATRIX(field, suffix, d0, d1) \ do { \ const int64_t e[2] = {(d0), (d1)}; \ snprintf(name, sizeof(name), "roberta.encoder.layer.%d." suffix, i); \ l->field = load_weight(sf, name, e, 2); \ if (!l->field) \ return -1; \ } while (0) #define LOAD_VECTOR(field, suffix, d0) \ do { \ const int64_t e[1] = {(d0)}; \ snprintf(name, sizeof(name), "roberta.encoder.layer.%d." suffix, i); \ l->field = load_weight(sf, name, e, 1); \ if (!l->field) \ return -1; \ } while (0) LOAD_MATRIX(wq, "attention.self.query.weight", hidden, hidden); LOAD_VECTOR(q_bias, "attention.self.query.bias", hidden); LOAD_MATRIX(wk, "attention.self.key.weight", hidden, hidden); LOAD_VECTOR(k_bias, "attention.self.key.bias", hidden); LOAD_MATRIX(wv, "attention.self.value.weight", hidden, hidden); LOAD_VECTOR(v_bias, "attention.self.value.bias", hidden); LOAD_MATRIX(wo, "attention.output.dense.weight", hidden, hidden); LOAD_VECTOR(o_bias, "attention.output.dense.bias", hidden); LOAD_VECTOR(input_norm, "attention.output.LayerNorm.weight", hidden); LOAD_VECTOR(attn_ln_bias, "attention.output.LayerNorm.bias", hidden); LOAD_MATRIX(up_proj, "intermediate.dense.weight", inter, hidden); LOAD_VECTOR(ffn_inter_bias, "intermediate.dense.bias", inter); LOAD_MATRIX(down_proj, "output.dense.weight", hidden, inter); LOAD_VECTOR(ffn_out_bias, "output.dense.bias", hidden); LOAD_VECTOR(post_attn_norm, "output.LayerNorm.weight", hidden); LOAD_VECTOR(ffn_ln_bias, "output.LayerNorm.bias", hidden); #undef LOAD_MATRIX #undef LOAD_VECTOR } return 0; } langdet_model_t *langdet_model_load(const char *model_dir) { if (!model_dir) return NULL; langdet_model_t *model = (langdet_model_t *)calloc(1, sizeof(langdet_model_t)); if (!model) return NULL; if (langdet_config_parse(&model->config, model_dir) != 0) goto fail; char path[1024]; int length = snprintf(path, sizeof(path), "%s/model.safetensors", model_dir); if (length < 0 || (size_t)length >= sizeof(path)) goto fail; model->safetensors = safetensors_open(path); if (!model->safetensors) { fprintf(stderr, "langdet_model_load: failed to open safetensors in %s\n", model_dir); goto fail; } if (load_xlm_roberta_weights(model, model->safetensors) != 0) goto fail; return model; fail: langdet_model_free(model); return NULL; } void langdet_model_free(langdet_model_t *model) { if (!model) return; safetensors_close(model->safetensors); free(model->weights.layers); free(model); } const langdet_config_t *langdet_model_config(const langdet_model_t *model) { return model ? &model->config : NULL; }