aboutsummaryrefslogtreecommitdiff
path: root/libffwd/cuda_sdpa.h
blob: a6ee80b27fc63dcc549a2f910a83af7e494382cd (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
/* cuDNN-backed fused scaled-dot-product attention for the CUDA backend.
 *
 * Wraps the cuDNN graph API (via the vendored cudnn-frontend headers) behind a
 * small C interface so cuda.cu can dispatch packed variable-length attention
 * to NVIDIA's fused flash kernels. libcudnn is dlopen'd at runtime: when it is
 * absent or too old, ffwd_sdpa_create returns NULL and the caller keeps the
 * built-in kernels. Builds without cuDNN headers compile this interface to
 * stubs (see cuda_sdpa.cu).
 */
#ifndef FFWD_CUDA_SDPA_H
#define FFWD_CUDA_SDPA_H

#include <cuda_runtime.h>
#include <stdint.h>

#ifdef __cplusplus
extern "C" {
#endif

typedef struct ffwd_sdpa ffwd_sdpa_t;

/* Returns NULL when fused SDPA is unavailable (built without cuDNN headers,
 * libcudnn missing at runtime, or an unsupported cuDNN/driver combination). */
ffwd_sdpa_t *ffwd_sdpa_create(void);
void ffwd_sdpa_free(ffwd_sdpa_t *s);

/* Fused SDPA forward over a packed variable-length batch.
 *
 * Layout (all device pointers, 16-bit elements; bf16 when is_f16 == 0, f16
 * otherwise):
 *   q, o: [total_tokens, hq * d], token-major, head then dim within a token
 *   k, v: [total_tokens, hkv * d] (compact GQA: one row per kv head)
 * Softmax and accumulation run in f32 inside the fused kernel.
 *
 * Sequence metadata (device int32, prepared by the caller):
 *   seq_len:    [batch]      per-sequence token counts (0 for padded entries)
 *   ragged_q:   [batch + 1]  prefix offsets in ELEMENTS of q/o rows
 *                            (offsets[b] * hq * d), last entry = total * hq * d
 *   ragged_kv:  [batch + 1]  same for k/v rows (offsets[b] * hkv * d)
 * batch and s_max are the graph dimensions and may exceed the live batch /
 * longest sequence (bucketing): padded batch entries carry seq_len 0 and a
 * repeated final ragged offset.
 *
 * Returns 0 on success; -1 on an execution error; -2 when this shape is
 * unsupported by the installed cuDNN (negative-cached, cheap to re-ask).
 * On nonzero the caller must fall back to the built-in kernels. */
int ffwd_sdpa_run(ffwd_sdpa_t *s,
                  cudaStream_t stream,
                  const void *q,
                  const void *k,
                  const void *v,
                  void *o,
                  const int32_t *seq_len,
                  const int32_t *ragged_q,
                  const int32_t *ragged_kv,
                  int batch,
                  int s_max,
                  int hq,
                  int hkv,
                  int d,
                  float scale,
                  int causal,
                  int is_f16);

#ifdef __cplusplus
}
#endif

#endif /* FFWD_CUDA_SDPA_H */