aboutsummaryrefslogtreecommitdiff
path: root/libffwd
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-07-02 19:44:08 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-07-02 19:44:08 +0200
commit6a10d3577232f1fd646a20f22a4a1ab4ec29f28a (patch)
treee4a077b064876ddb39144935fa256a4ab0692488 /libffwd
parent98a713e86c23bc6098bf7a5558507a9f27a7a1c1 (diff)
downloadffwd-6a10d3577232f1fd646a20f22a4a1ab4ec29f28a.tar.gz
ffwd-6a10d3577232f1fd646a20f22a4a1ab4ec29f28a.zip
cuda: add cuDNN fused attention path with runtime fallback
Diffstat (limited to 'libffwd')
-rw-r--r--libffwd/Makefile2
-rw-r--r--libffwd/cuda.cu195
-rw-r--r--libffwd/cuda_sdpa.cu343
-rw-r--r--libffwd/cuda_sdpa.h69
4 files changed, 603 insertions, 6 deletions
diff --git a/libffwd/Makefile b/libffwd/Makefile
index 2372697..10e6b14 100644
--- a/libffwd/Makefile
+++ b/libffwd/Makefile
@@ -30,7 +30,7 @@ ifeq ($(BACKEND),mlx)
CU_SRCS :=
else ifeq ($(BACKEND),cuda)
SRCS := $(CUDA_SRCS)
- CU_SRCS := cuda.cu
+ CU_SRCS := cuda.cu cuda_sdpa.cu
else
SRCS := $(BLAS_SRCS)
CU_SRCS :=
diff --git a/libffwd/cuda.cu b/libffwd/cuda.cu
index b4ee764..1884b73 100644
--- a/libffwd/cuda.cu
+++ b/libffwd/cuda.cu
@@ -3,6 +3,8 @@ extern "C" {
#include "model_types.h"
}
+#include "cuda_sdpa.h"
+
#include <cuda_runtime.h>
#include <cuda_bf16.h>
#include <cuda_fp16.h>
@@ -108,6 +110,18 @@ struct ffwd_cuda_ctx {
void *kexp_bf16; // BF16 K expansion for tensor-core streaming attention
void *kexp_f16; // FP16 K expansion for tensor-core streaming attention
float *vexp;
+ // cuDNN fused attention (cuda_sdpa.h). q16 is the packed 16-bit Q
+ // [total x q_dim]; sdpa_meta holds the per-forward sequence metadata
+ // (lengths + ragged offsets) in one device block, staged by
+ // cuda_upload_packed_inputs and shared by every layer.
+ ffwd_sdpa_t *sdpa;
+ void *q16;
+ int *sdpa_meta; // device: [seq_len bb][ragged_q bb+1][ragged_kv bb+1]
+ int *sdpa_meta_host; // persistent host mirror for the async upload
+ int sdpa_meta_cap; // capacity in int entries
+ int sdpa_batch_bucket;
+ int sdpa_smax_bucket;
+ int sdpa_kv_row; // K/V row width in elements the staged ragged_kv assumes
float *attn_scores;
void *attn_probs; // BF16 softmax output (bf16 weights)
long long attn_scores_elems;
@@ -518,7 +532,11 @@ rms_norm_kernel(OUT_T *out, const float *x, const float *weight, int rows, int d
// One grid covers Q and K: blockIdx.y < n_heads selects the Q head with
// q_norm/q_offset, the remaining blocks the K head with k_norm/k_offset.
+// q16, when non-NULL, receives a packed BF16 copy of the final (normed,
+// rotated) Q rows as [rows x n_heads*head_dim] for the fused-attention path;
+// the store is free here because the kernel already produces every Q element.
__global__ static void rms_norm_rope_qk_kernel(float *x,
+ __nv_bfloat16 *q16,
const float *q_norm,
const float *k_norm,
const int *positions,
@@ -545,6 +563,7 @@ __global__ static void rms_norm_rope_qk_kernel(float *x,
float sign = tid < half ? -1.0f : 1.0f;
const float *weight = head < n_heads ? q_norm : k_norm;
int base_offset = head < n_heads ? q_offset : k_offset;
+ int q_head = head < n_heads ? head : -1;
if (head >= n_heads)
head -= n_heads;
size_t base = (size_t)row * row_stride + base_offset + head * head_dim;
@@ -578,7 +597,11 @@ __global__ static void rms_norm_rope_qk_kernel(float *x,
a *= inv * weight[tid];
b *= inv * weight[pair_d];
}
- x[base + tid] = a * c + sign * b * s;
+ float r = a * c + sign * b * s;
+ x[base + tid] = r;
+ if (q16 && q_head >= 0)
+ q16[(size_t)row * n_heads * head_dim + (size_t)q_head * head_dim + tid] =
+ __float2bfloat16(r);
}
}
@@ -600,6 +623,22 @@ silu_mul_packed_kernel(OUT_T *out, int out_stride, const float *gate_up, int row
}
}
+// Packs the Q columns of the fused QKV projection into a contiguous 16-bit
+// [total x q_dim] tensor for the cuDNN fused-attention path (which needs Q in
+// the same dtype as K/V). The Qwen path gets this for free inside
+// rms_norm_rope_qk_kernel; this kernel serves the families without RoPE.
+template <typename IN_T, typename OUT_T>
+__global__ static void
+pack_q16_kernel(OUT_T *q16, const IN_T *qkv, int total, int q_dim, int qkv_dim, int q_offset) {
+ int i = blockIdx.x * blockDim.x + threadIdx.x;
+ int n = total * q_dim;
+ if (i < n) {
+ int row = i / q_dim;
+ int d = i - row * q_dim;
+ q16[i] = (OUT_T)(float)qkv[(size_t)row * qkv_dim + q_offset + d];
+ }
+}
+
// Expand K and V from the n_kv_heads layout to a contiguous [total x q_dim]
// per-query-head layout (GQA: each kv head repeated n_heads/n_kv_heads times)
// so the attention GEMMs can stride uniformly over query heads.
@@ -2824,6 +2863,20 @@ static int use_streaming_attention(const ffwd_config_t *c, int out_bf16, int hea
return (mode == 0 || mode == 8) && c->family == FFWD_FAMILY_QWEN3 && out_bf16 && head_dim == 128;
}
+// cuDNN fused attention (cuda_sdpa.h). Unset/1 dispatches the 16-bit flash
+// shapes to cuDNN whenever the runtime supports them; FFWD_CUDA_SDPA=0 forces
+// the built-in kernels (the A/B instrument and the rescue switch).
+static int sdpa_mode(void) {
+ static int init = 0, mode = 0;
+ if (!init) {
+ const char *e = getenv("FFWD_CUDA_SDPA");
+ if (e && (!strcmp(e, "0") || !strcmp(e, "off")))
+ mode = -1;
+ init = 1;
+ }
+ return mode;
+}
+
// Grow the F32 weight-widen scratch to hold `elems` floats. The memory-only
// BF16 path materializes one F32 weight at a time into this buffer.
static int ensure_weight_f32(ffwd_cuda_ctx_t *ctx, size_t elems) {
@@ -3354,6 +3407,15 @@ static int cuda_attention_gemm(ffwd_cuda_ctx_t *ctx,
int direct_kv_mode = bert_direct_kv_mode();
int use_bert_direct_kv = use_bert_flash && qkv_dtype != CUDA_DTYPE_F32 &&
qkv_dtype == out_dtype && direct_kv_mode >= 0;
+ // cuDNN fused attention takes exactly the shapes the built-in 16-bit flash
+ // kernels would run (same selection gates; FFWD_CUDA_SDPA=0 restores the
+ // hand-written kernels, and any cuDNN refusal falls back to them). It
+ // consumes the staged contiguous K/V, so the BERT direct-QKV read shortcut
+ // is bypassed while it is active.
+ int use_sdpa = sdpa_mode() >= 0 && ctx->sdpa && ctx->q16 && ctx->sdpa_batch_bucket > 0 &&
+ (use_tc_kexp_bf16 || use_bert_flash);
+ if (use_sdpa)
+ use_bert_direct_kv = 0;
int threads = 256;
size_t exp_count = (size_t)total * q_dim;
@@ -3406,6 +3468,47 @@ static int cuda_attention_gemm(ffwd_cuda_ctx_t *ctx,
return -1;
}
+ if (use_sdpa) {
+ // Q must match the K/V dtype: the Qwen path already wrote ctx->q16
+ // inside the RoPE kernel; the BERT path packs it here from the
+ // (possibly still F32) fused QKV projection.
+ if (c->family == FFWD_FAMILY_BERT) {
+ size_t n = (size_t)total * q_dim;
+ int blocks = (int)((n + threads - 1) / threads);
+ if (out_dtype == CUDA_DTYPE_F16) {
+ if (qkv_dtype == CUDA_DTYPE_F16)
+ pack_q16_kernel<<<blocks, threads, 0, ctx->stream>>>(
+ (__half *)ctx->q16, (const __half *)qkv_src, total, q_dim, qkv_dim, q_offset);
+ else
+ pack_q16_kernel<<<blocks, threads, 0, ctx->stream>>>(
+ (__half *)ctx->q16, (const float *)qkv_src, total, q_dim, qkv_dim, q_offset);
+ } else {
+ if (qkv_dtype == CUDA_DTYPE_BF16)
+ pack_q16_kernel<<<blocks, threads, 0, ctx->stream>>>(
+ (__nv_bfloat16 *)ctx->q16, (const __nv_bfloat16 *)qkv_src, total, q_dim,
+ qkv_dim, q_offset);
+ else
+ pack_q16_kernel<<<blocks, threads, 0, ctx->stream>>>(
+ (__nv_bfloat16 *)ctx->q16, (const float *)qkv_src, total, q_dim, qkv_dim,
+ q_offset);
+ }
+ if (launch_check() != 0)
+ return -1;
+ }
+ const void *ksrc = out_dtype == CUDA_DTYPE_F16 ? ctx->kexp_f16 : ctx->kexp_bf16;
+ int bb = ctx->sdpa_batch_bucket;
+ const int32_t *seq = (const int32_t *)ctx->sdpa_meta;
+ const int32_t *rq = seq + bb;
+ const int32_t *rkv = rq + bb + 1;
+ int hkv_eff = ctx->sdpa_kv_row / hd;
+ if (ffwd_sdpa_run(ctx->sdpa, ctx->stream, ctx->q16, ksrc, ctx->vexp, out_16, seq, rq, rkv, bb,
+ ctx->sdpa_smax_bucket, H, hkv_eff, hd, scale, causal,
+ out_dtype == CUDA_DTYPE_F16) == 0)
+ return 0;
+ // Unsupported shape or execution failure: the staged K/V and Q remain
+ // valid for the built-in kernels below.
+ }
+
if (use_bert_flash) {
int w = tc_kv_warps_or(8);
#define BERT_FLASH_LAUNCH(HDV, W) \
@@ -3765,7 +3868,12 @@ static void bert_cuda_plan(ffwd_cuda_ctx_t *ctx, int total, bert_cuda_plan_t *p)
int qkv16_mode = bert_qkv16_mode();
p->qkv16 = p->flash && p->cublaslt_avail && qkv16_mode >= 0 &&
(qkv16_mode > 0 || total >= BERT_QKV16_MIN_TOKENS);
- p->direct_kv = p->flash && p->qkv16 && bert_direct_kv_mode() >= 0;
+ // The cuDNN fused-attention path reads staged contiguous K/V, so while it
+ // is a candidate the direct packed-QKV read is not planned and the K/V
+ // expansion buffers stay allocated (the dispatch relies on them, including
+ // for its fallback).
+ p->direct_kv =
+ p->flash && p->qkv16 && bert_direct_kv_mode() >= 0 && !(ctx->sdpa && sdpa_mode() >= 0);
}
static int ensure_buffers(ffwd_cuda_ctx_t *ctx, int total, int batch, int max_seq) {
@@ -3793,6 +3901,10 @@ static int ensure_buffers(ffwd_cuda_ctx_t *ctx, int total, int batch, int max_se
int need_act_f16 = ctx->weights_f16;
int act_cols = 2 * c->intermediate_size;
int ffn_gate_up_cols = 2 * c->intermediate_size;
+ // Packed 16-bit Q for the cuDNN fused-attention path: any shape whose
+ // 16-bit flash kernel could dispatch to cuDNN needs it staged.
+ int need_q16 = ctx->sdpa && sdpa_mode() >= 0 &&
+ (is_bert ? bp.flash : (ctx->weights_bf16 && c->head_dim == 128));
if (is_bert) {
need_x_norm = 0;
@@ -3835,6 +3947,7 @@ static int ensure_buffers(ffwd_cuda_ctx_t *ctx, int total, int batch, int max_se
scratch_key |= need_act_f16 << 16;
scratch_key |= is_bert << 17;
scratch_key |= (bp.flash ? 1 : 0) << 18;
+ scratch_key |= need_q16 << 19;
if (total > ctx->seq_cap || scratch_key != ctx->scratch_key) {
cudaFree(ctx->x);
@@ -3853,6 +3966,7 @@ static int ensure_buffers(ffwd_cuda_ctx_t *ctx, int total, int batch, int max_se
cudaFree(ctx->kexp_bf16);
cudaFree(ctx->kexp_f16);
cudaFree(ctx->vexp);
+ cudaFree(ctx->q16);
cudaFree(ctx->act_bf16);
cudaFree(ctx->act_f16);
cudaFree(ctx->rope_cos);
@@ -3870,6 +3984,7 @@ static int ensure_buffers(ffwd_cuda_ctx_t *ctx, int total, int batch, int max_se
ctx->kexp = ctx->vexp = NULL;
ctx->kexp_bf16 = NULL;
ctx->kexp_f16 = NULL;
+ ctx->q16 = NULL;
ctx->act_bf16 = NULL;
ctx->act_f16 = NULL;
ctx->rope_cos = NULL;
@@ -3911,6 +4026,8 @@ static int ensure_buffers(ffwd_cuda_ctx_t *ctx, int total, int batch, int max_se
CUDA_CHECK(cudaMalloc(&ctx->kexp_f16, (size_t)total * c->q_dim * sizeof(__half)));
if (need_vexp)
CUDA_CHECK(cudaMalloc((void **)&ctx->vexp, (size_t)total * c->q_dim * vexp_elem_bytes));
+ if (need_q16)
+ CUDA_CHECK(cudaMalloc(&ctx->q16, (size_t)total * c->q_dim * sizeof(uint16_t)));
if (need_act_bf16)
CUDA_CHECK(cudaMalloc(&ctx->act_bf16, (size_t)total * act_cols * sizeof(__nv_bfloat16)));
if (need_act_f16)
@@ -4118,6 +4235,11 @@ static ffwd_cuda_ctx_t *cuda_ctx_from_model(ffwd_model_t *cpu, int own_cpu) {
ffwd_cuda_free(ctx);
return NULL;
}
+ // cuDNN fused attention; NULL (missing/old libcudnn or FFWD_CUDA_SDPA=0)
+ // keeps the built-in kernels. Only 16-bit weight paths can dispatch to it,
+ // so exact-F32 models skip loading libcudnn entirely.
+ if (sdpa_mode() >= 0 && (ctx->weights_bf16 || ctx->weights_f16))
+ ctx->sdpa = ffwd_sdpa_create();
// cuBLASLt handle + workspace + a small 16-bit bias scratch for the fused
// bias/GeLU epilogue GEMMs (the bias is cast to the output dtype per call).
ctx->lt_ws_bytes = (size_t)32 << 20;
@@ -4213,6 +4335,10 @@ void ffwd_cuda_free(ffwd_cuda_ctx_t *ctx) {
cudaFree(ctx->kexp_bf16);
cudaFree(ctx->kexp_f16);
cudaFree(ctx->vexp);
+ cudaFree(ctx->q16);
+ cudaFree(ctx->sdpa_meta);
+ free(ctx->sdpa_meta_host);
+ ffwd_sdpa_free(ctx->sdpa);
cudaFree(ctx->act_bf16);
cudaFree(ctx->act_f16);
cudaFree(ctx->weight_f32);
@@ -4243,6 +4369,60 @@ const ffwd_config_t *ffwd_cuda_config(const ffwd_cuda_ctx_t *ctx) {
// Pack inputs into the device scratch every forward shares: token ids, the
// per-sequence positions 0..L-1, and the batch offsets. Grows scratch via
// ensure_buffers. Returns the packed token count in *out_total, or -1 on error.
+// Stages the cuDNN fused-attention sequence metadata for one forward pass:
+// per-sequence lengths plus ragged element offsets for Q/O and K/V rows,
+// padded to a power-of-two batch bucket (padded entries are zero-length
+// sequences) so the per-shape graph cache stays small. Layer-invariant, so it
+// runs once per forward and every attention call reuses the device block.
+static int cuda_sdpa_stage_meta(ffwd_cuda_ctx_t *ctx, const int *h_offsets, int batch) {
+ const ffwd_config_t *c = &ctx->config;
+ int total = h_offsets[batch];
+ if ((long long)total * c->q_dim > INT_MAX)
+ return -1; // ragged offsets are int32 element counts
+ int max_L = 0;
+ for (int b = 0; b < batch; b++) {
+ int L = h_offsets[b + 1] - h_offsets[b];
+ if (L > max_L)
+ max_L = L;
+ }
+ int bb = 1;
+ while (bb < batch)
+ bb <<= 1;
+ int smax = 64;
+ while (smax < max_L)
+ smax <<= 1;
+ int kv_row = (c->family == FFWD_FAMILY_QWEN3 && qwen_compact_kv()) ? c->n_kv_heads * c->head_dim
+ : c->q_dim;
+ int need = bb + 2 * (bb + 1);
+ if (need > ctx->sdpa_meta_cap) {
+ cudaFree(ctx->sdpa_meta);
+ free(ctx->sdpa_meta_host);
+ ctx->sdpa_meta = NULL;
+ ctx->sdpa_meta_cap = 0;
+ ctx->sdpa_meta_host = (int *)malloc((size_t)need * sizeof(int));
+ if (!ctx->sdpa_meta_host)
+ return -1;
+ CUDA_CHECK(cudaMalloc((void **)&ctx->sdpa_meta, (size_t)need * sizeof(int)));
+ ctx->sdpa_meta_cap = need;
+ }
+ int *seq = ctx->sdpa_meta_host;
+ int *rq = seq + bb;
+ int *rkv = rq + bb + 1;
+ for (int b = 0; b < bb; b++)
+ seq[b] = b < batch ? h_offsets[b + 1] - h_offsets[b] : 0;
+ for (int b = 0; b <= bb; b++) {
+ int off = b <= batch ? h_offsets[b] : total;
+ rq[b] = off * c->q_dim;
+ rkv[b] = off * kv_row;
+ }
+ CUDA_CHECK(cudaMemcpyAsync(ctx->sdpa_meta, ctx->sdpa_meta_host, (size_t)need * sizeof(int),
+ cudaMemcpyHostToDevice, ctx->stream));
+ ctx->sdpa_batch_bucket = bb;
+ ctx->sdpa_smax_bucket = smax;
+ ctx->sdpa_kv_row = kv_row;
+ return 0;
+}
+
static int cuda_upload_packed_inputs(ffwd_cuda_ctx_t *ctx,
const ffwd_input_t *inputs,
int batch,
@@ -4296,6 +4476,11 @@ static int cuda_upload_packed_inputs(ffwd_cuda_ctx_t *ctx,
if (ce == cudaSuccess)
ce = cudaMemcpyAsync(ctx->offsets, h_offsets, (size_t)(batch + 1) * sizeof(int),
cudaMemcpyHostToDevice, ctx->stream);
+ // Metadata staging failure only disables the cuDNN attention path for this
+ // forward; the built-in kernels do not need it.
+ ctx->sdpa_batch_bucket = 0;
+ if (ce == cudaSuccess && ctx->q16)
+ (void)cuda_sdpa_stage_meta(ctx, h_offsets, batch);
free(h_offsets);
free(h_ids);
free(h_pos);
@@ -4842,9 +5027,9 @@ static int cuda_forward_batch(ffwd_cuda_ctx_t *ctx, const ffwd_input_t *inputs,
}
rms_norm_rope_qk_kernel<<<dim3(total, c->n_heads + c->n_kv_heads), 128, 0, ctx->stream>>>(
- ctx->qkv, l->q_norm, l->k_norm, ctx->positions, ctx->rope_cos, ctx->rope_sin, total,
- c->n_heads, c->n_kv_heads, c->head_dim, qkv_dim, q_offset, k_offset, c->qk_norm,
- c->rms_norm_eps);
+ ctx->qkv, (__nv_bfloat16 *)ctx->q16, l->q_norm, l->k_norm, ctx->positions, ctx->rope_cos,
+ ctx->rope_sin, total, c->n_heads, c->n_kv_heads, c->head_dim, qkv_dim, q_offset, k_offset,
+ c->qk_norm, c->rms_norm_eps);
if (launch_check() != 0)
return -1;
diff --git a/libffwd/cuda_sdpa.cu b/libffwd/cuda_sdpa.cu
new file mode 100644
index 0000000..d3f7d09
--- /dev/null
+++ b/libffwd/cuda_sdpa.cu
@@ -0,0 +1,343 @@
+/* cuDNN graph-API fused attention (see cuda_sdpa.h for the contract).
+ *
+ * Compiled against the vendored cudnn-frontend headers with
+ * NV_CUDNN_FRONTEND_USE_DYNAMIC_LOADING: libcudnn is dlopen'd on first use
+ * instead of linked, so the CUDA build carries no hard cuDNN dependency.
+ * When the build has no cuDNN headers (FFWD_CUDA_SDPA undefined) this file
+ * compiles to stubs that report the feature as unavailable.
+ *
+ * One graph is built per attention shape class (batch bucket, sequence
+ * bucket, head counts, head dim, mask, dtype) and cached; variable lengths
+ * inside a bucket are handled by the kernel itself through per-sequence
+ * lengths and ragged offsets, so the cache stays small (buckets are powers
+ * of two). Shapes the installed cuDNN cannot fuse are negative-cached and
+ * the caller falls back to the built-in kernels.
+ */
+
+#include "cuda_sdpa.h"
+
+#include <stdio.h>
+
+#ifndef FFWD_CUDA_SDPA
+
+extern "C" ffwd_sdpa_t *ffwd_sdpa_create(void) { return NULL; }
+extern "C" void ffwd_sdpa_free(ffwd_sdpa_t *) {}
+extern "C" int ffwd_sdpa_run(ffwd_sdpa_t *,
+ cudaStream_t,
+ const void *,
+ const void *,
+ const void *,
+ void *,
+ const int32_t *,
+ const int32_t *,
+ const int32_t *,
+ int,
+ int,
+ int,
+ int,
+ int,
+ float,
+ int,
+ int) {
+ return -2;
+}
+
+#else /* FFWD_CUDA_SDPA */
+
+# include <dlfcn.h>
+
+# include <cstdint>
+# include <cstdlib>
+# include <cstring>
+# include <map>
+# include <memory>
+# include <tuple>
+# include <unordered_map>
+
+# include <cudnn_frontend.h>
+
+namespace fe = cudnn_frontend;
+
+/* The dynamic-loading shim resolves every cuDNN symbol through this handle;
+ * the application owns its definition and initialization. */
+namespace cudnn_frontend {
+void *cudnn_dlhandle = nullptr;
+}
+
+namespace {
+
+/* Variant-pack tensor ids, fixed across all graphs. */
+enum : int64_t {
+ UID_Q = 1,
+ UID_K,
+ UID_V,
+ UID_O,
+ UID_SEQ_Q,
+ UID_SEQ_KV,
+ UID_RAGGED_Q,
+ UID_RAGGED_KV,
+ UID_RAGGED_O,
+};
+
+struct GraphKey {
+ int b, s, hq, hkv, d, causal, f16;
+ uint32_t scale_bits;
+ bool operator<(const GraphKey &o) const {
+ return std::tie(b, s, hq, hkv, d, causal, f16, scale_bits) <
+ std::tie(o.b, o.s, o.hq, o.hkv, o.d, o.causal, o.f16, o.scale_bits);
+ }
+};
+
+/* A built graph, or a negative entry (graph == nullptr) for shapes the
+ * installed cuDNN rejected. */
+struct GraphEntry {
+ std::shared_ptr<fe::graph::Graph> graph;
+ int64_t workspace_bytes = 0;
+};
+
+} // namespace
+
+struct ffwd_sdpa {
+ cudnnHandle_t handle = nullptr;
+ std::map<GraphKey, GraphEntry> cache;
+ void *workspace = nullptr;
+ size_t workspace_cap = 0;
+ int exec_error_reported = 0;
+ int unsupported_reported = 0;
+};
+
+/* Builds the fused SDPA graph for one shape class. Returns nullptr when the
+ * installed cuDNN does not support it (the caller negative-caches). */
+static std::shared_ptr<fe::graph::Graph> sdpa_build_graph(cudnnHandle_t handle, const GraphKey &k) {
+ auto graph = std::make_shared<fe::graph::Graph>();
+ graph->set_io_data_type(k.f16 ? fe::DataType_t::HALF : fe::DataType_t::BFLOAT16)
+ .set_intermediate_data_type(fe::DataType_t::FLOAT)
+ .set_compute_data_type(fe::DataType_t::FLOAT);
+
+ const int64_t b = k.b, hq = k.hq, hkv = k.hkv, d = k.d, s = k.s;
+
+ /* Packed token-major ("THD") tensors: within one sequence the token
+ * stride is heads*d and the head stride is d; per-sequence start offsets
+ * come from the ragged-offset tensors, which make the nominal batch
+ * stride irrelevant for addressing (it still describes the padded
+ * bucket). Ragged offsets are element counts, int32, shape (b+1). */
+ auto ragged_q = graph->tensor(fe::graph::Tensor_attributes()
+ .set_name("ragged_q")
+ .set_uid(UID_RAGGED_Q)
+ .set_dim({b + 1, 1, 1, 1})
+ .set_stride({1, 1, 1, 1})
+ .set_data_type(fe::DataType_t::INT32));
+ auto ragged_kv = graph->tensor(fe::graph::Tensor_attributes()
+ .set_name("ragged_kv")
+ .set_uid(UID_RAGGED_KV)
+ .set_dim({b + 1, 1, 1, 1})
+ .set_stride({1, 1, 1, 1})
+ .set_data_type(fe::DataType_t::INT32));
+ auto ragged_o = graph->tensor(fe::graph::Tensor_attributes()
+ .set_name("ragged_o")
+ .set_uid(UID_RAGGED_O)
+ .set_dim({b + 1, 1, 1, 1})
+ .set_stride({1, 1, 1, 1})
+ .set_data_type(fe::DataType_t::INT32));
+
+ auto Q = graph->tensor(fe::graph::Tensor_attributes()
+ .set_name("Q")
+ .set_uid(UID_Q)
+ .set_dim({b, hq, s, d})
+ .set_stride({s * hq * d, d, hq * d, 1})
+ .set_ragged_offset(ragged_q));
+ auto K = graph->tensor(fe::graph::Tensor_attributes()
+ .set_name("K")
+ .set_uid(UID_K)
+ .set_dim({b, hkv, s, d})
+ .set_stride({s * hkv * d, d, hkv * d, 1})
+ .set_ragged_offset(ragged_kv));
+ auto V = graph->tensor(fe::graph::Tensor_attributes()
+ .set_name("V")
+ .set_uid(UID_V)
+ .set_dim({b, hkv, s, d})
+ .set_stride({s * hkv * d, d, hkv * d, 1})
+ .set_ragged_offset(ragged_kv));
+
+ auto seq_q = graph->tensor(fe::graph::Tensor_attributes()
+ .set_name("seq_q")
+ .set_uid(UID_SEQ_Q)
+ .set_dim({b, 1, 1, 1})
+ .set_stride({1, 1, 1, 1})
+ .set_data_type(fe::DataType_t::INT32));
+ auto seq_kv = graph->tensor(fe::graph::Tensor_attributes()
+ .set_name("seq_kv")
+ .set_uid(UID_SEQ_KV)
+ .set_dim({b, 1, 1, 1})
+ .set_stride({1, 1, 1, 1})
+ .set_data_type(fe::DataType_t::INT32));
+
+ float scale;
+ std::memcpy(&scale, &k.scale_bits, sizeof(scale));
+ auto opts = fe::graph::SDPA_attributes()
+ .set_name("ffwd_sdpa")
+ .set_generate_stats(false)
+ .set_attn_scale(scale)
+ .set_padding_mask(true)
+ .set_seq_len_q(seq_q)
+ .set_seq_len_kv(seq_kv);
+ if (k.causal)
+ opts.set_causal_mask(true);
+
+ /* Ragged inputs are only expressible by the flash engines (the classic
+ * multi-op graph on SM80+, the unified fused op on newer cuDNN), so an
+ * unfusable shape fails the build here instead of silently running a
+ * materialized fallback. Known gates: GQA+ragged needs cuDNN >= 9.6,
+ * ragged on Ampere/Ada (sm8x) needs >= 9.18. */
+ auto [O, stats] = graph->sdpa(Q, K, V, opts);
+ (void)stats;
+ O->set_output(true)
+ .set_uid(UID_O)
+ .set_dim({b, hq, s, d})
+ .set_stride({s * hq * d, d, hq * d, 1})
+ .set_ragged_offset(ragged_o);
+
+ if (!graph->build(handle, {fe::HeurMode_t::A}).is_good())
+ return nullptr;
+ return graph;
+}
+
+extern "C" ffwd_sdpa_t *ffwd_sdpa_create(void) {
+ const char *lib = getenv("FFWD_CUDNN_LIB");
+ void *dl = nullptr;
+ if (lib && lib[0])
+ dl = dlopen(lib, RTLD_NOW | RTLD_LOCAL);
+ if (!dl)
+ dl = dlopen("libcudnn.so.9", RTLD_NOW | RTLD_LOCAL);
+ if (!dl)
+ dl = dlopen("libcudnn.so", RTLD_NOW | RTLD_LOCAL);
+ if (!dl) {
+ fprintf(stderr, "ffwd-cuda: libcudnn not found, using built-in attention kernels\n");
+ return NULL;
+ }
+ cudnn_frontend::cudnn_dlhandle = dl;
+
+ ffwd_sdpa_t *s = NULL;
+ try {
+ /* The fused SDPA support surface used here (ragged THD, GQA head
+ * counts, padding mask) is stable from cuDNN 9 on. */
+ if (fe::detail::get_backend_version() < 90000) {
+ fprintf(stderr, "ffwd-cuda: cuDNN %s too old (need >= 9.0), using built-in kernels\n",
+ fe::detail::get_backend_version_string().c_str());
+ return NULL;
+ }
+ s = new ffwd_sdpa();
+ if (fe::detail::create_handle(&s->handle) != CUDNN_STATUS_SUCCESS) {
+ delete s;
+ return NULL;
+ }
+ } catch (...) {
+ delete s;
+ return NULL;
+ }
+ return s;
+}
+
+extern "C" void ffwd_sdpa_free(ffwd_sdpa_t *s) {
+ if (!s)
+ return;
+ s->cache.clear();
+ if (s->workspace)
+ cudaFree(s->workspace);
+ if (s->handle)
+ fe::detail::destroy_handle(s->handle);
+ delete s;
+}
+
+extern "C" int ffwd_sdpa_run(ffwd_sdpa_t *s,
+ cudaStream_t stream,
+ const void *q,
+ const void *k,
+ const void *v,
+ void *o,
+ const int32_t *seq_len,
+ const int32_t *ragged_q,
+ const int32_t *ragged_kv,
+ int batch,
+ int s_max,
+ int hq,
+ int hkv,
+ int d,
+ float scale,
+ int causal,
+ int is_f16) {
+ if (!s)
+ return -2;
+ GraphKey key;
+ key.b = batch;
+ key.s = s_max;
+ key.hq = hq;
+ key.hkv = hkv;
+ key.d = d;
+ key.causal = causal;
+ key.f16 = is_f16;
+ std::memcpy(&key.scale_bits, &scale, sizeof(scale));
+
+ try {
+ auto it = s->cache.find(key);
+ if (it == s->cache.end()) {
+ GraphEntry e;
+ e.graph = sdpa_build_graph(s->handle, key);
+ if (e.graph && !e.graph->get_workspace_size(e.workspace_bytes).is_good())
+ e.graph = nullptr;
+ if (!e.graph && !s->unsupported_reported) {
+ s->unsupported_reported = 1;
+ fprintf(stderr,
+ "ffwd-cuda: cuDNN %s cannot fuse this attention shape "
+ "(hq=%d hkv=%d d=%d causal=%d), using built-in kernels\n",
+ fe::detail::get_backend_version_string().c_str(), hq, hkv, d, causal);
+ }
+ it = s->cache.emplace(key, std::move(e)).first;
+ }
+ if (!it->second.graph)
+ return -2;
+
+ if ((size_t)it->second.workspace_bytes > s->workspace_cap) {
+ if (s->workspace)
+ cudaFree(s->workspace);
+ s->workspace = NULL;
+ s->workspace_cap = 0;
+ if (cudaMalloc(&s->workspace, (size_t)it->second.workspace_bytes) != cudaSuccess)
+ return -1;
+ s->workspace_cap = (size_t)it->second.workspace_bytes;
+ }
+
+ if (fe::detail::set_stream(s->handle, stream) != CUDNN_STATUS_SUCCESS)
+ return -1;
+
+ std::unordered_map<int64_t, void *> pack = {
+ {UID_Q, const_cast<void *>(q)},
+ {UID_K, const_cast<void *>(k)},
+ {UID_V, const_cast<void *>(v)},
+ {UID_O, o},
+ {UID_SEQ_Q, const_cast<int32_t *>(seq_len)},
+ {UID_SEQ_KV, const_cast<int32_t *>(seq_len)},
+ {UID_RAGGED_Q, const_cast<int32_t *>(ragged_q)},
+ {UID_RAGGED_KV, const_cast<int32_t *>(ragged_kv)},
+ {UID_RAGGED_O, const_cast<int32_t *>(ragged_q)},
+ };
+ if (!it->second.graph->execute(s->handle, pack, s->workspace).is_good()) {
+ if (!s->exec_error_reported) {
+ s->exec_error_reported = 1;
+ fprintf(stderr, "ffwd-cuda: cuDNN SDPA execution failed, using built-in kernels\n");
+ }
+ return -1;
+ }
+ } catch (const std::exception &ex) {
+ if (!s->exec_error_reported) {
+ s->exec_error_reported = 1;
+ fprintf(stderr, "ffwd-cuda: cuDNN SDPA error: %s\n", ex.what());
+ }
+ return -1;
+ } catch (...) {
+ return -1;
+ }
+ return 0;
+}
+
+#endif /* FFWD_CUDA_SDPA */
diff --git a/libffwd/cuda_sdpa.h b/libffwd/cuda_sdpa.h
new file mode 100644
index 0000000..a6ee80b
--- /dev/null
+++ b/libffwd/cuda_sdpa.h
@@ -0,0 +1,69 @@
+/* cuDNN-backed fused scaled-dot-product attention for the CUDA backend.
+ *
+ * Wraps the cuDNN graph API (via the vendored cudnn-frontend headers) behind a
+ * small C interface so cuda.cu can dispatch packed variable-length attention
+ * to NVIDIA's fused flash kernels. libcudnn is dlopen'd at runtime: when it is
+ * absent or too old, ffwd_sdpa_create returns NULL and the caller keeps the
+ * built-in kernels. Builds without cuDNN headers compile this interface to
+ * stubs (see cuda_sdpa.cu).
+ */
+#ifndef FFWD_CUDA_SDPA_H
+#define FFWD_CUDA_SDPA_H
+
+#include <cuda_runtime.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct ffwd_sdpa ffwd_sdpa_t;
+
+/* Returns NULL when fused SDPA is unavailable (built without cuDNN headers,
+ * libcudnn missing at runtime, or an unsupported cuDNN/driver combination). */
+ffwd_sdpa_t *ffwd_sdpa_create(void);
+void ffwd_sdpa_free(ffwd_sdpa_t *s);
+
+/* Fused SDPA forward over a packed variable-length batch.
+ *
+ * Layout (all device pointers, 16-bit elements; bf16 when is_f16 == 0, f16
+ * otherwise):
+ * q, o: [total_tokens, hq * d], token-major, head then dim within a token
+ * k, v: [total_tokens, hkv * d] (compact GQA: one row per kv head)
+ * Softmax and accumulation run in f32 inside the fused kernel.
+ *
+ * Sequence metadata (device int32, prepared by the caller):
+ * seq_len: [batch] per-sequence token counts (0 for padded entries)
+ * ragged_q: [batch + 1] prefix offsets in ELEMENTS of q/o rows
+ * (offsets[b] * hq * d), last entry = total * hq * d
+ * ragged_kv: [batch + 1] same for k/v rows (offsets[b] * hkv * d)
+ * batch and s_max are the graph dimensions and may exceed the live batch /
+ * longest sequence (bucketing): padded batch entries carry seq_len 0 and a
+ * repeated final ragged offset.
+ *
+ * Returns 0 on success; -1 on an execution error; -2 when this shape is
+ * unsupported by the installed cuDNN (negative-cached, cheap to re-ask).
+ * On nonzero the caller must fall back to the built-in kernels. */
+int ffwd_sdpa_run(ffwd_sdpa_t *s,
+ cudaStream_t stream,
+ const void *q,
+ const void *k,
+ const void *v,
+ void *o,
+ const int32_t *seq_len,
+ const int32_t *ragged_q,
+ const int32_t *ragged_kv,
+ int batch,
+ int s_max,
+ int hq,
+ int hkv,
+ int d,
+ float scale,
+ int causal,
+ int is_f16);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* FFWD_CUDA_SDPA_H */