aboutsummaryrefslogtreecommitdiff
path: root/embed_internal.h
blob: 53debee60f38a17af07a12d60834110b8a89bcc6 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
#ifndef PPLX_EMBED_INTERNAL_H
#define PPLX_EMBED_INTERNAL_H

#include "embed.h"
#include "qwen_safetensors.h"

typedef struct {
    safetensor_dtype_t dtype;
    const void *data;
} pplx_weight_ref_t;

typedef struct {
    pplx_weight_ref_t wq;        /* [q_dim,   hidden] */
    pplx_weight_ref_t wk;        /* [kv_dim,  hidden] */
    pplx_weight_ref_t wv;        /* [kv_dim,  hidden] */
    pplx_weight_ref_t wo;        /* [hidden,  q_dim]  */
    const float *q_norm;         /* [head_dim] */
    const float *k_norm;         /* [head_dim] */
    const float *input_norm;     /* [hidden] */
    const float *post_attn_norm; /* [hidden] */
    pplx_weight_ref_t gate_proj; /* [intermediate, hidden] */
    pplx_weight_ref_t up_proj;   /* [intermediate, hidden] */
    pplx_weight_ref_t down_proj; /* [hidden, intermediate] */
} pplx_layer_t;

typedef struct {
    pplx_weight_ref_t embed_tokens; /* [vocab_size, hidden] */
    pplx_layer_t *layers;           /* [n_layers] heap-allocated */
    const float *norm;              /* [hidden] */
} pplx_weights_t;

struct pplx_model {
    pplx_config_t  config;
    pplx_weights_t weights;
    void *safetensors;          /* multi_safetensors_t*, keeps mmap alive */
    int layer_start;            /* loaded transformer range, inclusive */
    int layer_end;              /* loaded transformer range, exclusive */
};

struct pplx_workspace {
    const pplx_model_t *model;  /* immutable model this workspace belongs to */
    int    buf_seq_cap;
    float *x;                   /* [seq, hidden]       */
    float *x_norm;              /* [seq, hidden]       */
    float *q;                   /* [seq, q_dim]        */
    float *k;                   /* [seq, kv_dim]       */
    float *v;                   /* [seq, kv_dim]       */
    float *attn_out;            /* [seq, q_dim]        */
    float *proj_out;            /* [seq, hidden]       */
    float *ffn_gate;            /* [seq, intermediate] */
    float *ffn_up;              /* [seq, intermediate] */
    float *attn_scores;         /* reusable BLAS attention score tiles */
    size_t attn_scores_bytes;

    int   *offsets;             /* [batch + 1] packed sequence offsets */
    int    offsets_cap;

    /* RoPE cosine/sine cache [n_pos, head_dim], grown lazily. */
    float *rope_cos;
    float *rope_sin;
    int    rope_cache_cap;

    /* F32 view of one BF16 weight matrix for SGEMM dispatch, grown to the
     * largest projection. Workspace-owned so concurrent workspaces on one
     * model never share conversion scratch. */
    float *bf16_widen;
    size_t bf16_widen_count;
};

#endif /* PPLX_EMBED_INTERNAL_H */