aboutsummaryrefslogtreecommitdiff
path: root/src/config.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-16 20:30:55 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-16 23:38:17 +0200
commit1ea03a65ed2a836005ea7a0061db100e470cc69d (patch)
tree9274145ea702cdc5d9574e5dc3b2dcaa0bd726ae /src/config.c
parent7500042ac2c33e0501e3d2ebae5a423e61cc39d1 (diff)
downloadffwd-1ea03a65ed2a836005ea7a0061db100e470cc69d.tar.gz
ffwd-1ea03a65ed2a836005ea7a0061db100e470cc69d.zip
all: add SentencePiece XLM-R support
Diffstat (limited to 'src/config.c')
-rw-r--r--src/config.c20
1 files changed, 18 insertions, 2 deletions
diff --git a/src/config.c b/src/config.c
index f693843..a33a135 100644
--- a/src/config.c
+++ b/src/config.c
@@ -209,6 +209,8 @@ int embed_config_parse(embed_config_t *cfg, const char *model_dir) {
cfg->layer_norm_eps = (float)json_get_double(buf, "layer_norm_eps", 1e-12);
cfg->rope_theta = (float)json_get_double(buf, "rope_theta", 1000000.0);
cfg->max_position_embeddings = json_get_int(buf, "max_position_embeddings", 0);
+ cfg->position_id_offset = 0;
+ cfg->type_vocab_size = json_get_int(buf, "type_vocab_size", 2);
cfg->family = EMBED_FAMILY_QWEN3;
cfg->qk_norm = 1;
cfg->qkv_bias = 0;
@@ -226,13 +228,25 @@ int embed_config_parse(embed_config_t *cfg, const char *model_dir) {
cfg->qk_norm = 0;
cfg->qkv_bias = 1;
cfg->attention_mode = EMBED_ATTENTION_CAUSAL;
- } else if (json_string_equals(buf, "model_type", "bert")) {
+ } else if (json_string_equals(buf, "model_type", "bert") ||
+ json_string_equals(buf, "model_type", "roberta") ||
+ json_string_equals(buf, "model_type", "xlm-roberta")) {
cfg->family = EMBED_FAMILY_BERT;
cfg->qk_norm = 0;
cfg->qkv_bias = 1; /* q/k/v/o and both dense layers carry bias */
cfg->attention_mode = EMBED_ATTENTION_BIDIRECTIONAL;
cfg->n_kv_heads = cfg->n_heads; /* full multi-head attention, no GQA */
cfg->head_dim = cfg->n_heads > 0 ? cfg->hidden_size / cfg->n_heads : 0;
+ if (json_string_equals(buf, "model_type", "roberta") ||
+ json_string_equals(buf, "model_type", "xlm-roberta")) {
+ int pad_id = json_get_int(buf, "pad_token_id", 1);
+ if (pad_id < 0 || pad_id == INT_MAX) {
+ fprintf(stderr, "embed_config: invalid RoBERTa pad_token_id\n");
+ free(buf);
+ return -1;
+ }
+ cfg->position_id_offset = pad_id + 1;
+ }
/* hidden_act selects the feed-forward GeLU curve. gelu_new and
* gelu_pytorch_tanh are the same tanh approximation; everything else
* (gelu, absent) is the exact erf GeLU the released encoders use. */
@@ -240,7 +254,9 @@ int embed_config_parse(embed_config_t *cfg, const char *model_dir) {
json_string_equals(buf, "hidden_act", "gelu_pytorch_tanh")) {
cfg->ffn_act = EMBED_ACT_GELU_TANH;
}
- if (cfg->max_position_embeddings <= 0 || !isfinite(cfg->layer_norm_eps) ||
+ if (cfg->max_position_embeddings <= 0 ||
+ cfg->position_id_offset >= cfg->max_position_embeddings ||
+ cfg->type_vocab_size <= 0 || !isfinite(cfg->layer_norm_eps) ||
cfg->layer_norm_eps <= 0.0f) {
fprintf(stderr, "embed_config: invalid BERT layer_norm_eps/max_position_embeddings\n");
free(buf);