diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 13:11:00 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 13:11:00 +0200 |
| commit | 37cbaef9ceb828c667da8f398027ba5415ee7475 (patch) | |
| tree | 6103e92a9030766e78a35df00f7a1155d0d0ecf5 /liblangdet/config.c | |
| parent | cf7d8169ec12cb3adf90d4ff78b3257e74985302 (diff) | |
| download | langdet-37cbaef9ceb828c667da8f398027ba5415ee7475.tar.gz langdet-37cbaef9ceb828c667da8f398027ba5415ee7475.zip | |
Rename CPU library API and sources to langdet
Diffstat (limited to 'liblangdet/config.c')
| -rw-r--r-- | liblangdet/config.c | 67 |
1 files changed, 67 insertions, 0 deletions
diff --git a/liblangdet/config.c b/liblangdet/config.c new file mode 100644 index 0000000..f22a9d7 --- /dev/null +++ b/liblangdet/config.c @@ -0,0 +1,67 @@ +#include "config.h" + +#include <yyjson.h> + +#include <limits.h> +#include <math.h> +#include <stdio.h> + +static int json_get_int(yyjson_val *obj, const char *key, int fallback) { + yyjson_val *v = yyjson_obj_get(obj, key); + if (!v) + return fallback; + if (!yyjson_is_int(v)) + return -1; + int64_t n = yyjson_get_sint(v); + return n >= 0 && n <= INT_MAX ? (int)n : -1; +} + +int langdet_config_parse(langdet_config_t *cfg, const char *model_dir) { + char path[1024]; + int n = snprintf(path, sizeof(path), "%s/config.json", model_dir); + if (n < 0 || (size_t)n >= sizeof(path)) { + fprintf(stderr, "langdet_config: model path too long\n"); + return -1; + } + + yyjson_read_err error; + yyjson_doc *doc = yyjson_read_file(path, 0, NULL, &error); + if (!doc) { + fprintf(stderr, "langdet_config: cannot read %s: %s\n", path, error.msg); + return -1; + } + yyjson_val *root = yyjson_doc_get_root(doc); + yyjson_val *causal = yyjson_obj_get(root, "is_causal"); + if (!yyjson_equals_str(yyjson_obj_get(root, "model_type"), "xlm-roberta") || + !yyjson_equals_str(yyjson_obj_get(root, "hidden_act"), "gelu") || + (causal && !yyjson_is_false(causal))) { + fprintf(stderr, "langdet_config: expected bidirectional XLM-R with GeLU activation\n"); + yyjson_doc_free(doc); + return -1; + } + + cfg->hidden_size = json_get_int(root, "hidden_size", 0); + cfg->n_layers = json_get_int(root, "num_hidden_layers", 0); + cfg->n_heads = json_get_int(root, "num_attention_heads", 0); + cfg->head_dim = cfg->n_heads > 0 ? cfg->hidden_size / cfg->n_heads : 0; + cfg->intermediate_size = json_get_int(root, "intermediate_size", 0); + cfg->vocab_size = json_get_int(root, "vocab_size", 0); + cfg->max_position_embeddings = json_get_int(root, "max_position_embeddings", 0); + cfg->type_vocab_size = json_get_int(root, "type_vocab_size", 1); + yyjson_val *eps = yyjson_obj_get(root, "layer_norm_eps"); + cfg->layer_norm_eps = eps ? (float)yyjson_get_num(eps) : 1e-5f; + int pad_id = json_get_int(root, "pad_token_id", 1); + cfg->position_id_offset = pad_id >= 0 && pad_id < INT_MAX ? pad_id + 1 : -1; + yyjson_doc_free(doc); + + if (cfg->hidden_size <= 0 || cfg->n_layers <= 0 || cfg->n_layers > LANGDET_MAX_LAYERS || + cfg->n_heads <= 0 || cfg->head_dim <= 0 || cfg->hidden_size % cfg->n_heads != 0 || + cfg->intermediate_size <= 0 || cfg->vocab_size <= 0 || cfg->type_vocab_size <= 0 || + cfg->position_id_offset < 0 || + cfg->max_position_embeddings <= cfg->position_id_offset || + !isfinite(cfg->layer_norm_eps) || cfg->layer_norm_eps <= 0.0f) { + fprintf(stderr, "langdet_config: invalid encoder dimensions or normalization in %s\n", path); + return -1; + } + return 0; +} |
