aboutsummaryrefslogtreecommitdiff
path: root/embed.h
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-16 09:26:20 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-16 09:26:20 +0200
commitc20279050f88d9c24187c3e402912596e1f54587 (patch)
tree2750fc95cca51258c1ceb31cfcce6c2a46cb2bc7 /embed.h
parent849c571a8e79d92c1fd3dabecf0fb3621aa4f59b (diff)
downloadffwd-c20279050f88d9c24187c3e402912596e1f54587.tar.gz
ffwd-c20279050f88d9c24187c3e402912596e1f54587.zip
all: support BERT tanh-GeLU (gelu_new / gelu_pytorch_tanh)
Diffstat (limited to 'embed.h')
-rw-r--r--embed.h9
1 files changed, 9 insertions, 0 deletions
diff --git a/embed.h b/embed.h
index 860576e..c7381bd 100644
--- a/embed.h
+++ b/embed.h
@@ -48,8 +48,17 @@ typedef enum {
EMBED_FAMILY_BERT = 1, /* LayerNorm + learned positions + GeLU encoder block */
} embed_family_t;
+/* BERT feed-forward activation. The released MiniLM/BGE encoders use exact erf
+ * GeLU; some encoders publish the tanh approximation (hidden_act gelu_new or
+ * gelu_pytorch_tanh), which is a different curve and must be matched exactly. */
+typedef enum {
+ EMBED_ACT_GELU_ERF = 0, /* 0.5 x (1 + erf(x / sqrt(2))) */
+ EMBED_ACT_GELU_TANH = 1, /* 0.5 x (1 + tanh(sqrt(2/pi) (x + 0.044715 x^3))) */
+} embed_act_t;
+
typedef struct {
embed_family_t family;
+ embed_act_t ffn_act; /* BERT feed-forward GeLU variant; ERF for Qwen/SwiGLU */
int hidden_size;
int n_layers;
int n_heads;