diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-15 15:40:22 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-15 15:40:22 +0200 |
| commit | 898466e9d0c19c6ac5bfa4f3f02e13f70a0e73c4 (patch) | |
| tree | fe8cbd924a4d6da59c0df2b21d928d4a50a27b8c | |
| parent | ceecbea701e3ac882ce5f1693204152cc75c0d79 (diff) | |
| download | ffwd-898466e9d0c19c6ac5bfa4f3f02e13f70a0e73c4.tar.gz ffwd-898466e9d0c19c6ac5bfa4f3f02e13f70a0e73c4.zip | |
cpu: add the BERT encoder family forward and loader
| -rw-r--r-- | embed.c | 245 | ||||
| -rw-r--r-- | embed.h | 8 | ||||
| -rw-r--r-- | embed_config.c | 22 | ||||
| -rw-r--r-- | embed_internal.h | 18 |
4 files changed, 276 insertions, 17 deletions
@@ -488,6 +488,99 @@ static int ensure_offsets(embed_workspace_t *ws, int batch) { * Model / workspace load and free * ======================================================================== */ +/* BERT-family weight loader. The tensor names differ entirely from the Qwen + * block (encoder.layer.N.*, embeddings.*), so the family gets its own entry + * point. Reused embed_layer_t slots carry their BERT meaning: wo is the + * attention output dense, input_norm/post_attn_norm are the two block LayerNorm + * weights, up_proj/down_proj are the two FFN matrices (see embed_internal.h). */ +static int +load_bert_weights(embed_model_t *model, multi_safetensors_t *ms, int layer_start, int layer_end) { + const embed_config_t *cfg = &model->config; + embed_weights_t *w = &model->weights; + char name[256]; + + /* sentence-transformers checkpoints store the BERT submodule unprefixed; + * raw HF checkpoints prefix every tensor with "bert.". */ + const char *p; + if (multi_safetensors_find(ms, "embeddings.word_embeddings.weight", NULL)) + p = ""; + else if (multi_safetensors_find(ms, "bert.embeddings.word_embeddings.weight", NULL)) + p = "bert."; + else { + fprintf(stderr, "embed_model_load: BERT word embeddings not found\n"); + return -1; + } + + int hidden = cfg->hidden_size, inter = cfg->intermediate_size; + + if (layer_start == 0) { + const int64_t emb_shape[2] = {cfg->vocab_size, hidden}; + snprintf(name, sizeof(name), "%sembeddings.word_embeddings.weight", p); + w->embed_tokens = load_weight_direct(ms, name, emb_shape, 2); + const int64_t pos_shape[2] = {cfg->max_position_embeddings, hidden}; + snprintf(name, sizeof(name), "%sembeddings.position_embeddings.weight", p); + w->position_embeddings = load_norm_f32(ms, name, pos_shape, 2); + const int64_t type_shape[2] = {2, hidden}; + snprintf(name, sizeof(name), "%sembeddings.token_type_embeddings.weight", p); + w->token_type_embeddings = load_norm_f32(ms, name, type_shape, 2); + const int64_t ln_shape[1] = {hidden}; + snprintf(name, sizeof(name), "%sembeddings.LayerNorm.weight", p); + w->embed_ln_w = load_norm_f32(ms, name, ln_shape, 1); + snprintf(name, sizeof(name), "%sembeddings.LayerNorm.bias", p); + w->embed_ln_b = load_norm_f32(ms, name, ln_shape, 1); + if (!w->embed_tokens.data || !w->position_embeddings || !w->token_type_embeddings || + !w->embed_ln_w || !w->embed_ln_b) + return -1; + } + + w->layers = (embed_layer_t *)calloc(cfg->n_layers, sizeof(embed_layer_t)); + if (!w->layers) + return -1; + + for (int i = layer_start; i < layer_end; i++) { + embed_layer_t *l = &w->layers[i]; + +#define BERT_W(field, suffix, d0, d1) \ + do { \ + const int64_t e[2] = {(d0), (d1)}; \ + snprintf(name, sizeof(name), "%sencoder.layer.%d." suffix, p, i); \ + l->field = load_weight_direct(ms, name, e, 2); \ + if (!l->field.data) \ + return -1; \ + } while (0) +#define BERT_V(field, suffix, d0) \ + do { \ + const int64_t e[1] = {(d0)}; \ + snprintf(name, sizeof(name), "%sencoder.layer.%d." suffix, p, i); \ + l->field = load_norm_f32(ms, name, e, 1); \ + if (!l->field) \ + return -1; \ + } while (0) + + BERT_W(wq, "attention.self.query.weight", hidden, hidden); + BERT_V(q_bias, "attention.self.query.bias", hidden); + BERT_W(wk, "attention.self.key.weight", hidden, hidden); + BERT_V(k_bias, "attention.self.key.bias", hidden); + BERT_W(wv, "attention.self.value.weight", hidden, hidden); + BERT_V(v_bias, "attention.self.value.bias", hidden); + BERT_W(wo, "attention.output.dense.weight", hidden, hidden); + BERT_V(o_bias, "attention.output.dense.bias", hidden); + BERT_V(input_norm, "attention.output.LayerNorm.weight", hidden); + BERT_V(attn_ln_bias, "attention.output.LayerNorm.bias", hidden); + BERT_W(up_proj, "intermediate.dense.weight", inter, hidden); + BERT_V(ffn_inter_bias, "intermediate.dense.bias", inter); + BERT_W(down_proj, "output.dense.weight", hidden, inter); + BERT_V(ffn_out_bias, "output.dense.bias", hidden); + BERT_V(post_attn_norm, "output.LayerNorm.weight", hidden); + BERT_V(ffn_ln_bias, "output.LayerNorm.bias", hidden); + +#undef BERT_W +#undef BERT_V + } + /* BERT has no final norm: w->norm stays NULL, and pooling skips it. */ + return 0; +} + static embed_model_t * model_load_range_ex(const char *model_dir, int layer_start, int layer_end, int allow_late) { if (!allow_late && model_dir_has_late_projection(model_dir)) { @@ -533,6 +626,15 @@ model_load_range_ex(const char *model_dir, int layer_start, int layer_end, int a } model->safetensors = ms; + if (cfg->family == EMBED_FAMILY_BERT) { + if (load_bert_weights(model, ms, layer_start, layer_end) != 0) + goto fail; + if (embed_verbose >= 1) + fprintf(stderr, "embed_model_load: BERT layers [%d, %d) loaded (%d-dim)\n", layer_start, + layer_end, cfg->hidden_size); + return model; + } + const char *weight_prefix = multi_safetensors_weight_prefix(ms, "embed_tokens.weight"); if (!weight_prefix) { fprintf(stderr, "embed_model_load: neither embed_tokens.weight nor " @@ -649,9 +751,18 @@ void embed_model_free(embed_model_t *model) { free((void *)l->k_norm); free((void *)l->input_norm); free((void *)l->post_attn_norm); + free((void *)l->o_bias); + free((void *)l->attn_ln_bias); + free((void *)l->ffn_inter_bias); + free((void *)l->ffn_out_bias); + free((void *)l->ffn_ln_bias); } } free((void *)model->weights.norm); + free((void *)model->weights.position_embeddings); + free((void *)model->weights.token_type_embeddings); + free((void *)model->weights.embed_ln_w); + free((void *)model->weights.embed_ln_b); if (model->safetensors) multi_safetensors_close((multi_safetensors_t *)model->safetensors); free(model->weights.layers); @@ -1158,6 +1269,97 @@ static int forward_packed_inplace(const embed_model_t *model, 0, model->config.n_layers, apply_final_norm); } +/* BERT-family forward: token + learned-position + token-type embeddings and an + * embedding LayerNorm, then post-norm encoder layers (bias on every projection, + * GeLU feed-forward, no RoPE, no per-head norm). Bidirectional attention is + * block-diagonal over the packed batch, so each sequence's positions restart at + * 0. Leaves the final hidden states in ws->x; pooling reads them as-is because + * the last layer's LayerNorm is already the model's final normalization. */ +static int forward_packed_bert(const embed_model_t *model, + embed_workspace_t *ws, + const embed_input_t *inputs, + int batch, + const int *offsets, + int total_seq, + int max_seq) { + if (!model || !ws || ws->model != model || !inputs || batch <= 0 || !offsets || + total_seq <= 0 || max_seq <= 0) + return -1; + + const embed_config_t *cfg = &model->config; + const embed_weights_t *w = &model->weights; + int hidden = cfg->hidden_size; + int n_heads = cfg->n_heads; + int head_dim = cfg->head_dim; + int inter = cfg->intermediate_size; + float eps = cfg->layer_norm_eps; + + if (!w->embed_tokens.data || !w->position_embeddings || !w->token_type_embeddings || + !w->embed_ln_w || !w->embed_ln_b) + return -1; + if (max_seq > cfg->max_position_embeddings) { + fprintf(stderr, "embed: sequence length %d exceeds max_position_embeddings %d\n", max_seq, + cfg->max_position_embeddings); + return -1; + } + if (ensure_buffers(ws, cfg, total_seq) != 0) + return -1; + ensure_attention_scores(ws, offsets, batch); + + float *x = ws->x; + float *q_buf = ws->q, *k_buf = ws->k, *v_buf = ws->v; + float *attn_out = ws->attn_out, *proj_out = ws->proj_out, *ffn_up = ws->ffn_up; + + /* Embeddings: token + learned absolute position + token-type[0], then the + * embedding LayerNorm over every packed token. */ + for (int b = 0; b < batch; b++) { + int start = offsets[b]; + for (int i = 0; i < inputs[b].n_tokens; i++) { + int id = inputs[b].ids[i]; + if (id < 0 || id >= cfg->vocab_size) { + fprintf(stderr, "embed: invalid token id %d at input %d position %d\n", id, b, i); + return -1; + } + float *row = x + (size_t)(start + i) * hidden; + copy_weight_row(row, &w->embed_tokens, (size_t)id, hidden); + qwen_add_inplace(row, w->position_embeddings + (size_t)i * hidden, hidden); + qwen_add_inplace(row, w->token_type_embeddings, hidden); + } + } + qwen_layer_norm(x, x, w->embed_ln_w, w->embed_ln_b, total_seq, hidden, eps); + + float scale = 1.0f / sqrtf((float)head_dim); + + for (int layer = 0; layer < cfg->n_layers; layer++) { + const embed_layer_t *l = &w->layers[layer]; + + /* Self-attention (post-norm): projections read x directly. */ + linear_qkv_weight(ws, q_buf, k_buf, v_buf, x, &l->wq, &l->wk, &l->wv, total_seq, hidden, + hidden, hidden); + add_bias_rows(q_buf, l->q_bias, total_seq, hidden); + add_bias_rows(k_buf, l->k_bias, total_seq, hidden); + add_bias_rows(v_buf, l->v_bias, total_seq, hidden); + + model->attention(attn_out, q_buf, k_buf, v_buf, offsets, batch, n_heads, n_heads, head_dim, + scale, ws->attn_scores, ws->attn_scores_bytes); + + linear_nobias_weight(ws, proj_out, attn_out, &l->wo, total_seq, hidden, hidden); + add_bias_rows(proj_out, l->o_bias, total_seq, hidden); + qwen_add_inplace(x, proj_out, total_seq * hidden); + qwen_layer_norm(x, x, l->input_norm, l->attn_ln_bias, total_seq, hidden, eps); + + /* Feed-forward (post-norm): dense -> GeLU -> dense. */ + linear_nobias_weight(ws, ffn_up, x, &l->up_proj, total_seq, hidden, inter); + add_bias_rows(ffn_up, l->ffn_inter_bias, total_seq, inter); + qwen_gelu_inplace(ffn_up, total_seq * inter); + linear_nobias_weight(ws, proj_out, ffn_up, &l->down_proj, total_seq, inter, hidden); + add_bias_rows(proj_out, l->ffn_out_bias, total_seq, hidden); + qwen_add_inplace(x, proj_out, total_seq * hidden); + qwen_layer_norm(x, x, l->post_attn_norm, l->ffn_ln_bias, total_seq, hidden, eps); + } + return 0; +} + static int pool_embeddings(const embed_model_t *model, const embed_workspace_t *ws, const int *offsets, @@ -1166,9 +1368,11 @@ static int pool_embeddings(const embed_model_t *model, const embed_config_t *cfg = &model->config; int hidden = cfg->hidden_size; float eps = cfg->rms_norm_eps; + /* Qwen pools pre-final-norm states and applies the final RMSNorm here; BERT + * pools already-normed states (no final norm), signalled by a NULL norm. */ const float *norm_weight = model->weights.norm; const float *x = ws->x; - if (hidden <= 0 || !norm_weight || !x) + if (hidden <= 0 || !x) return -1; for (int b = 0; b < batch; b++) { @@ -1177,29 +1381,36 @@ static int pool_embeddings(const embed_model_t *model, int len = end - start; float *emb = out_embeddings + (size_t)b * hidden; - memset(emb, 0, (size_t)hidden * sizeof(float)); if (cfg->pooling_mode == EMBED_POOL_LAST_TOKEN || cfg->pooling_mode == EMBED_POOL_CLS) { /* Single-token pooling: CLS takes the first token, last-token the - * final one. The model's final RMSNorm is applied to that one row. */ + * final one. */ int idx = cfg->pooling_mode == EMBED_POOL_CLS ? start : end - 1; const float *row = x + (size_t)idx * hidden; - float sum_sq = 0.0f; - for (int d = 0; d < hidden; d++) - sum_sq += row[d] * row[d]; - float rms_inv = 1.0f / sqrtf(sum_sq / (float)hidden + eps); - for (int d = 0; d < hidden; d++) - emb[d] = row[d] * rms_inv * norm_weight[d]; - } else { - for (int i = start; i < end; i++) { - const float *row = x + (size_t)i * hidden; - + if (norm_weight) { float sum_sq = 0.0f; for (int d = 0; d < hidden; d++) sum_sq += row[d] * row[d]; - float rms_inv = 1.0f / sqrtf(sum_sq / (float)hidden + eps); for (int d = 0; d < hidden; d++) - emb[d] += row[d] * rms_inv * norm_weight[d]; + emb[d] = row[d] * rms_inv * norm_weight[d]; + } else { + memcpy(emb, row, (size_t)hidden * sizeof(float)); + } + } else { + memset(emb, 0, (size_t)hidden * sizeof(float)); + for (int i = start; i < end; i++) { + const float *row = x + (size_t)i * hidden; + if (norm_weight) { + float sum_sq = 0.0f; + for (int d = 0; d < hidden; d++) + sum_sq += row[d] * row[d]; + float rms_inv = 1.0f / sqrtf(sum_sq / (float)hidden + eps); + for (int d = 0; d < hidden; d++) + emb[d] += row[d] * rms_inv * norm_weight[d]; + } else { + for (int d = 0; d < hidden; d++) + emb[d] += row[d]; + } } float inv_len = 1.0f / (float)len; for (int d = 0; d < hidden; d++) @@ -1391,7 +1602,9 @@ int embed_model_encode_batch(const embed_model_t *model, if (build_offsets(inputs, batch, 1, offsets, &total_seq, &max_seq) != 0) return -1; - int rc = forward_packed_inplace(model, ws, inputs, batch, offsets, total_seq, max_seq, 0); + int rc = model->config.family == EMBED_FAMILY_BERT + ? forward_packed_bert(model, ws, inputs, batch, offsets, total_seq, max_seq) + : forward_packed_inplace(model, ws, inputs, batch, offsets, total_seq, max_seq, 0); if (rc == 0) rc = pool_embeddings(model, ws, offsets, batch, out_embeddings); @@ -43,7 +43,13 @@ typedef enum { EMBED_POOL_CLS = 2, /* first token; the encoder-family [CLS] sentence vector */ } embed_pooling_mode_t; +typedef enum { + EMBED_FAMILY_QWEN3 = 0, /* RMSNorm + RoPE + SwiGLU decoder block (pplx, Qwen2/3) */ + EMBED_FAMILY_BERT = 1, /* LayerNorm + learned positions + GeLU encoder block */ +} embed_family_t; + typedef struct { + embed_family_t family; int hidden_size; int n_layers; int n_heads; @@ -53,7 +59,9 @@ typedef struct { int kv_dim; /* n_kv_heads * head_dim */ int intermediate_size; int vocab_size; + int max_position_embeddings; /* BERT learned position table size */ float rms_norm_eps; + float layer_norm_eps; /* BERT LayerNorm epsilon */ float rope_theta; int qk_norm; int qkv_bias; diff --git a/embed_config.c b/embed_config.c index fc07334..03838dd 100644 --- a/embed_config.c +++ b/embed_config.c @@ -206,7 +206,10 @@ int embed_config_parse(embed_config_t *cfg, const char *model_dir) { cfg->intermediate_size = json_get_int(buf, "intermediate_size", 0); cfg->vocab_size = json_get_int(buf, "vocab_size", EMBED_VOCAB_SIZE); cfg->rms_norm_eps = (float)json_get_double(buf, "rms_norm_eps", 1e-6); + cfg->layer_norm_eps = (float)json_get_double(buf, "layer_norm_eps", 1e-12); cfg->rope_theta = (float)json_get_double(buf, "rope_theta", 1000000.0); + cfg->max_position_embeddings = json_get_int(buf, "max_position_embeddings", 0); + cfg->family = EMBED_FAMILY_QWEN3; cfg->qk_norm = 1; cfg->qkv_bias = 0; cfg->attention_mode = EMBED_ATTENTION_BIDIRECTIONAL; @@ -223,6 +226,25 @@ int embed_config_parse(embed_config_t *cfg, const char *model_dir) { cfg->qk_norm = 0; cfg->qkv_bias = 1; cfg->attention_mode = EMBED_ATTENTION_CAUSAL; + } else if (json_string_equals(buf, "model_type", "bert")) { + cfg->family = EMBED_FAMILY_BERT; + cfg->qk_norm = 0; + cfg->qkv_bias = 1; /* q/k/v/o and both dense layers carry bias */ + cfg->attention_mode = EMBED_ATTENTION_BIDIRECTIONAL; + cfg->n_kv_heads = cfg->n_heads; /* full multi-head attention, no GQA */ + cfg->head_dim = cfg->n_heads > 0 ? cfg->hidden_size / cfg->n_heads : 0; + if (json_string_equals(buf, "hidden_act", "gelu_new") || + json_string_equals(buf, "hidden_act", "gelu_pytorch_tanh")) { + fprintf(stderr, "embed_config: BERT tanh-GeLU is not supported yet; need exact gelu\n"); + free(buf); + return -1; + } + if (cfg->max_position_embeddings <= 0 || !isfinite(cfg->layer_norm_eps) || + cfg->layer_norm_eps <= 0.0f) { + fprintf(stderr, "embed_config: invalid BERT layer_norm_eps/max_position_embeddings\n"); + free(buf); + return -1; + } } json_bool_t is_causal = json_get_bool(buf, "is_causal"); diff --git a/embed_internal.h b/embed_internal.h index 0536692..39efbc4 100644 --- a/embed_internal.h +++ b/embed_internal.h @@ -24,12 +24,28 @@ typedef struct { embed_weight_ref_t gate_proj; /* [intermediate, hidden] */ embed_weight_ref_t up_proj; /* [intermediate, hidden] */ embed_weight_ref_t down_proj; /* [hidden, intermediate] */ + /* BERT family (all NULL for Qwen3). The attention projections reuse + * wq/wk/wv/wo and q/k/v_bias; the two FFN matrices reuse up_proj + * (intermediate.dense) and down_proj (output.dense); the two block + * LayerNorm weights reuse input_norm (post-attention) and post_attn_norm + * (post-FFN). These fields add the biases those reused slots lack. */ + const float *o_bias; /* attention output dense bias [hidden] */ + const float *attn_ln_bias; /* post-attention LayerNorm bias [hidden] */ + const float *ffn_inter_bias; /* intermediate.dense bias [intermediate] */ + const float *ffn_out_bias; /* output.dense bias [hidden] */ + const float *ffn_ln_bias; /* post-FFN LayerNorm bias [hidden] */ } embed_layer_t; typedef struct { embed_weight_ref_t embed_tokens; /* [vocab_size, hidden] */ embed_layer_t *layers; /* [n_layers] heap-allocated */ - const float *norm; /* [hidden] */ + const float *norm; /* [hidden]; NULL for BERT (no final norm) */ + /* BERT family (NULL for Qwen3): learned absolute position embeddings, + * token-type (segment) embeddings, and the embedding LayerNorm. */ + const float *position_embeddings; /* [max_position_embeddings, hidden] */ + const float *token_type_embeddings; /* [2, hidden] */ + const float *embed_ln_w; /* embedding LayerNorm weight [hidden] */ + const float *embed_ln_b; /* embedding LayerNorm bias [hidden] */ } embed_weights_t; typedef void (*embed_attention_fn)(float *out, |
