aboutsummaryrefslogtreecommitdiff
path: root/liblangdet/config.c
blob: f22a9d7e3864f5644632fa9b332732819f5ace79 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
#include "config.h"

#include <yyjson.h>

#include <limits.h>
#include <math.h>
#include <stdio.h>

static int json_get_int(yyjson_val *obj, const char *key, int fallback) {
    yyjson_val *v = yyjson_obj_get(obj, key);
    if (!v)
        return fallback;
    if (!yyjson_is_int(v))
        return -1;
    int64_t n = yyjson_get_sint(v);
    return n >= 0 && n <= INT_MAX ? (int)n : -1;
}

int langdet_config_parse(langdet_config_t *cfg, const char *model_dir) {
    char path[1024];
    int n = snprintf(path, sizeof(path), "%s/config.json", model_dir);
    if (n < 0 || (size_t)n >= sizeof(path)) {
        fprintf(stderr, "langdet_config: model path too long\n");
        return -1;
    }

    yyjson_read_err error;
    yyjson_doc *doc = yyjson_read_file(path, 0, NULL, &error);
    if (!doc) {
        fprintf(stderr, "langdet_config: cannot read %s: %s\n", path, error.msg);
        return -1;
    }
    yyjson_val *root = yyjson_doc_get_root(doc);
    yyjson_val *causal = yyjson_obj_get(root, "is_causal");
    if (!yyjson_equals_str(yyjson_obj_get(root, "model_type"), "xlm-roberta") ||
        !yyjson_equals_str(yyjson_obj_get(root, "hidden_act"), "gelu") ||
        (causal && !yyjson_is_false(causal))) {
        fprintf(stderr, "langdet_config: expected bidirectional XLM-R with GeLU activation\n");
        yyjson_doc_free(doc);
        return -1;
    }

    cfg->hidden_size = json_get_int(root, "hidden_size", 0);
    cfg->n_layers = json_get_int(root, "num_hidden_layers", 0);
    cfg->n_heads = json_get_int(root, "num_attention_heads", 0);
    cfg->head_dim = cfg->n_heads > 0 ? cfg->hidden_size / cfg->n_heads : 0;
    cfg->intermediate_size = json_get_int(root, "intermediate_size", 0);
    cfg->vocab_size = json_get_int(root, "vocab_size", 0);
    cfg->max_position_embeddings = json_get_int(root, "max_position_embeddings", 0);
    cfg->type_vocab_size = json_get_int(root, "type_vocab_size", 1);
    yyjson_val *eps = yyjson_obj_get(root, "layer_norm_eps");
    cfg->layer_norm_eps = eps ? (float)yyjson_get_num(eps) : 1e-5f;
    int pad_id = json_get_int(root, "pad_token_id", 1);
    cfg->position_id_offset = pad_id >= 0 && pad_id < INT_MAX ? pad_id + 1 : -1;
    yyjson_doc_free(doc);

    if (cfg->hidden_size <= 0 || cfg->n_layers <= 0 || cfg->n_layers > LANGDET_MAX_LAYERS ||
        cfg->n_heads <= 0 || cfg->head_dim <= 0 || cfg->hidden_size % cfg->n_heads != 0 ||
        cfg->intermediate_size <= 0 || cfg->vocab_size <= 0 || cfg->type_vocab_size <= 0 ||
        cfg->position_id_offset < 0 ||
        cfg->max_position_embeddings <= cfg->position_id_offset ||
        !isfinite(cfg->layer_norm_eps) || cfg->layer_norm_eps <= 0.0f) {
        fprintf(stderr, "langdet_config: invalid encoder dimensions or normalization in %s\n", path);
        return -1;
    }
    return 0;
}