1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
|
#ifndef PPLX_EMBED_INTERNAL_H
#define PPLX_EMBED_INTERNAL_H
#include "embed.h"
#include "qwen_safetensors.h"
typedef struct {
safetensor_dtype_t dtype;
const void *data;
} pplx_weight_ref_t;
typedef struct {
pplx_weight_ref_t wq; /* [q_dim, hidden] */
pplx_weight_ref_t wk; /* [kv_dim, hidden] */
pplx_weight_ref_t wv; /* [kv_dim, hidden] */
pplx_weight_ref_t wo; /* [hidden, q_dim] */
const float *q_norm; /* [head_dim] */
const float *k_norm; /* [head_dim] */
const float *input_norm; /* [hidden] */
const float *post_attn_norm; /* [hidden] */
pplx_weight_ref_t gate_proj; /* [intermediate, hidden] */
pplx_weight_ref_t up_proj; /* [intermediate, hidden] */
pplx_weight_ref_t down_proj; /* [hidden, intermediate] */
} pplx_layer_t;
typedef struct {
pplx_weight_ref_t embed_tokens; /* [vocab_size, hidden] */
pplx_layer_t *layers; /* [n_layers] heap-allocated */
const float *norm; /* [hidden] */
} pplx_weights_t;
struct pplx_model {
pplx_config_t config;
pplx_weights_t weights;
void *safetensors; /* multi_safetensors_t*, keeps mmap alive */
int layer_start; /* loaded transformer range, inclusive */
int layer_end; /* loaded transformer range, exclusive */
};
struct pplx_workspace {
const pplx_model_t *model; /* immutable model this workspace belongs to */
int buf_seq_cap;
float *x; /* [seq, hidden] */
float *x_norm; /* [seq, hidden] */
float *q; /* [seq, q_dim] */
float *k; /* [seq, kv_dim] */
float *v; /* [seq, kv_dim] */
float *attn_out; /* [seq, q_dim] */
float *proj_out; /* [seq, hidden] */
float *ffn_gate; /* [seq, intermediate] */
float *ffn_up; /* [seq, intermediate] */
float *attn_scores; /* reusable BLAS attention score tiles */
size_t attn_scores_bytes;
int *offsets; /* [batch + 1] packed sequence offsets */
int offsets_cap;
/* RoPE cosine/sine cache [n_pos, head_dim], grown lazily. */
float *rope_cos;
float *rope_sin;
int rope_cache_cap;
/* F32 view of one BF16 weight matrix for SGEMM dispatch, grown to the
* largest projection. Workspace-owned so concurrent workspaces on one
* model never share conversion scratch. */
float *bf16_widen;
size_t bf16_widen_count;
};
#endif /* PPLX_EMBED_INTERNAL_H */
|