1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
|
#ifndef EMBED_INTERNAL_H
#define EMBED_INTERNAL_H
#include "embed.h"
#include "safetensors.h"
typedef struct {
safetensor_dtype_t dtype;
const void *data;
} embed_weight_ref_t;
typedef struct {
embed_weight_ref_t wq; /* [q_dim, hidden] */
embed_weight_ref_t wk; /* [kv_dim, hidden] */
embed_weight_ref_t wv; /* [kv_dim, hidden] */
embed_weight_ref_t wo; /* [hidden, q_dim] */
const float *q_bias; /* [q_dim], optional */
const float *k_bias; /* [kv_dim], optional */
const float *v_bias; /* [kv_dim], optional */
const float *q_norm; /* [head_dim] */
const float *k_norm; /* [head_dim] */
const float *input_norm; /* [hidden] */
const float *post_attn_norm; /* [hidden] */
embed_weight_ref_t gate_proj; /* [intermediate, hidden] */
embed_weight_ref_t up_proj; /* [intermediate, hidden] */
embed_weight_ref_t down_proj; /* [hidden, intermediate] */
/* BERT family (all NULL for Qwen3). The attention projections reuse
* wq/wk/wv/wo and q/k/v_bias; the two FFN matrices reuse up_proj
* (intermediate.dense) and down_proj (output.dense); the two block
* LayerNorm weights reuse input_norm (post-attention) and post_attn_norm
* (post-FFN). These fields add the biases those reused slots lack. */
const float *o_bias; /* attention output dense bias [hidden] */
const float *attn_ln_bias; /* post-attention LayerNorm bias [hidden] */
const float *ffn_inter_bias; /* intermediate.dense bias [intermediate] */
const float *ffn_out_bias; /* output.dense bias [hidden] */
const float *ffn_ln_bias; /* post-FFN LayerNorm bias [hidden] */
} embed_layer_t;
typedef struct {
embed_weight_ref_t embed_tokens; /* [vocab_size, hidden] */
embed_layer_t *layers; /* [n_layers] heap-allocated */
const float *norm; /* [hidden]; NULL for BERT (no final norm) */
/* BERT family (NULL for Qwen3): learned absolute position embeddings,
* token-type (segment) embeddings, and the embedding LayerNorm. */
const float *position_embeddings; /* [max_position_embeddings, hidden] */
const float *token_type_embeddings; /* [2, hidden] */
const float *embed_ln_w; /* embedding LayerNorm weight [hidden] */
const float *embed_ln_b; /* embedding LayerNorm bias [hidden] */
} embed_weights_t;
typedef void (*embed_attention_fn)(float *out,
const float *Q,
const float *K,
const float *V,
const int *offsets,
int batch,
int n_heads,
int n_kv_heads,
int head_dim,
float scale,
float *scratch,
size_t scratch_bytes);
struct embed_model {
embed_config_t config;
embed_weights_t weights;
void *safetensors; /* multi_safetensors_t*, keeps mmap alive */
int layer_start; /* loaded transformer range, inclusive */
int layer_end; /* loaded transformer range, exclusive */
embed_attention_fn attention;
};
struct embed_workspace {
const embed_model_t *model; /* immutable model this workspace belongs to */
int buf_seq_cap;
float *x; /* [seq, hidden] */
float *x_norm; /* [seq, hidden] */
float *q; /* [seq, q_dim] */
float *k; /* [seq, kv_dim] */
float *v; /* [seq, kv_dim] */
float *attn_out; /* [seq, q_dim] */
float *proj_out; /* [seq, hidden] */
float *ffn_gate; /* [seq, intermediate] */
float *ffn_up; /* [seq, intermediate] */
float *attn_scores; /* reusable BLAS attention score tiles */
size_t attn_scores_bytes;
int *offsets; /* [batch + 1] packed sequence offsets */
int offsets_cap;
/* RoPE cosine/sine cache [n_pos, head_dim], grown lazily. */
float *rope_cos;
float *rope_sin;
int rope_cache_cap;
/* F32 view of one BF16 weight matrix for SGEMM dispatch, grown to the
* largest projection. Workspace-owned so concurrent workspaces on one
* model never share conversion scratch. */
float *bf16_widen;
size_t bf16_widen_count;
};
/* Internal accessors for the late-interaction model. GPU backends build a device
* context from an already-loaded CPU late model because the model dir is a late
* snapshot that the pooled loader rejects; these expose the base model and the
* projection weight without making struct embed_late_model public. */
embed_model_t *embed_late_model_base(const embed_late_model_t *model);
const embed_weight_ref_t *embed_late_model_projection(const embed_late_model_t *model);
#endif /* EMBED_INTERNAL_H */
|