diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-07-02 19:44:08 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-07-02 19:44:08 +0200 |
| commit | 6a10d3577232f1fd646a20f22a4a1ab4ec29f28a (patch) | |
| tree | e4a077b064876ddb39144935fa256a4ab0692488 /libffwd | |
| parent | 98a713e86c23bc6098bf7a5558507a9f27a7a1c1 (diff) | |
| download | ffwd-6a10d3577232f1fd646a20f22a4a1ab4ec29f28a.tar.gz ffwd-6a10d3577232f1fd646a20f22a4a1ab4ec29f28a.zip | |
cuda: add cuDNN fused attention path with runtime fallback
Diffstat (limited to 'libffwd')
| -rw-r--r-- | libffwd/Makefile | 2 | ||||
| -rw-r--r-- | libffwd/cuda.cu | 195 | ||||
| -rw-r--r-- | libffwd/cuda_sdpa.cu | 343 | ||||
| -rw-r--r-- | libffwd/cuda_sdpa.h | 69 |
4 files changed, 603 insertions, 6 deletions
diff --git a/libffwd/Makefile b/libffwd/Makefile index 2372697..10e6b14 100644 --- a/libffwd/Makefile +++ b/libffwd/Makefile @@ -30,7 +30,7 @@ ifeq ($(BACKEND),mlx) CU_SRCS := else ifeq ($(BACKEND),cuda) SRCS := $(CUDA_SRCS) - CU_SRCS := cuda.cu + CU_SRCS := cuda.cu cuda_sdpa.cu else SRCS := $(BLAS_SRCS) CU_SRCS := diff --git a/libffwd/cuda.cu b/libffwd/cuda.cu index b4ee764..1884b73 100644 --- a/libffwd/cuda.cu +++ b/libffwd/cuda.cu @@ -3,6 +3,8 @@ extern "C" { #include "model_types.h" } +#include "cuda_sdpa.h" + #include <cuda_runtime.h> #include <cuda_bf16.h> #include <cuda_fp16.h> @@ -108,6 +110,18 @@ struct ffwd_cuda_ctx { void *kexp_bf16; // BF16 K expansion for tensor-core streaming attention void *kexp_f16; // FP16 K expansion for tensor-core streaming attention float *vexp; + // cuDNN fused attention (cuda_sdpa.h). q16 is the packed 16-bit Q + // [total x q_dim]; sdpa_meta holds the per-forward sequence metadata + // (lengths + ragged offsets) in one device block, staged by + // cuda_upload_packed_inputs and shared by every layer. + ffwd_sdpa_t *sdpa; + void *q16; + int *sdpa_meta; // device: [seq_len bb][ragged_q bb+1][ragged_kv bb+1] + int *sdpa_meta_host; // persistent host mirror for the async upload + int sdpa_meta_cap; // capacity in int entries + int sdpa_batch_bucket; + int sdpa_smax_bucket; + int sdpa_kv_row; // K/V row width in elements the staged ragged_kv assumes float *attn_scores; void *attn_probs; // BF16 softmax output (bf16 weights) long long attn_scores_elems; @@ -518,7 +532,11 @@ rms_norm_kernel(OUT_T *out, const float *x, const float *weight, int rows, int d // One grid covers Q and K: blockIdx.y < n_heads selects the Q head with // q_norm/q_offset, the remaining blocks the K head with k_norm/k_offset. +// q16, when non-NULL, receives a packed BF16 copy of the final (normed, +// rotated) Q rows as [rows x n_heads*head_dim] for the fused-attention path; +// the store is free here because the kernel already produces every Q element. __global__ static void rms_norm_rope_qk_kernel(float *x, + __nv_bfloat16 *q16, const float *q_norm, const float *k_norm, const int *positions, @@ -545,6 +563,7 @@ __global__ static void rms_norm_rope_qk_kernel(float *x, float sign = tid < half ? -1.0f : 1.0f; const float *weight = head < n_heads ? q_norm : k_norm; int base_offset = head < n_heads ? q_offset : k_offset; + int q_head = head < n_heads ? head : -1; if (head >= n_heads) head -= n_heads; size_t base = (size_t)row * row_stride + base_offset + head * head_dim; @@ -578,7 +597,11 @@ __global__ static void rms_norm_rope_qk_kernel(float *x, a *= inv * weight[tid]; b *= inv * weight[pair_d]; } - x[base + tid] = a * c + sign * b * s; + float r = a * c + sign * b * s; + x[base + tid] = r; + if (q16 && q_head >= 0) + q16[(size_t)row * n_heads * head_dim + (size_t)q_head * head_dim + tid] = + __float2bfloat16(r); } } @@ -600,6 +623,22 @@ silu_mul_packed_kernel(OUT_T *out, int out_stride, const float *gate_up, int row } } +// Packs the Q columns of the fused QKV projection into a contiguous 16-bit +// [total x q_dim] tensor for the cuDNN fused-attention path (which needs Q in +// the same dtype as K/V). The Qwen path gets this for free inside +// rms_norm_rope_qk_kernel; this kernel serves the families without RoPE. +template <typename IN_T, typename OUT_T> +__global__ static void +pack_q16_kernel(OUT_T *q16, const IN_T *qkv, int total, int q_dim, int qkv_dim, int q_offset) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + int n = total * q_dim; + if (i < n) { + int row = i / q_dim; + int d = i - row * q_dim; + q16[i] = (OUT_T)(float)qkv[(size_t)row * qkv_dim + q_offset + d]; + } +} + // Expand K and V from the n_kv_heads layout to a contiguous [total x q_dim] // per-query-head layout (GQA: each kv head repeated n_heads/n_kv_heads times) // so the attention GEMMs can stride uniformly over query heads. @@ -2824,6 +2863,20 @@ static int use_streaming_attention(const ffwd_config_t *c, int out_bf16, int hea return (mode == 0 || mode == 8) && c->family == FFWD_FAMILY_QWEN3 && out_bf16 && head_dim == 128; } +// cuDNN fused attention (cuda_sdpa.h). Unset/1 dispatches the 16-bit flash +// shapes to cuDNN whenever the runtime supports them; FFWD_CUDA_SDPA=0 forces +// the built-in kernels (the A/B instrument and the rescue switch). +static int sdpa_mode(void) { + static int init = 0, mode = 0; + if (!init) { + const char *e = getenv("FFWD_CUDA_SDPA"); + if (e && (!strcmp(e, "0") || !strcmp(e, "off"))) + mode = -1; + init = 1; + } + return mode; +} + // Grow the F32 weight-widen scratch to hold `elems` floats. The memory-only // BF16 path materializes one F32 weight at a time into this buffer. static int ensure_weight_f32(ffwd_cuda_ctx_t *ctx, size_t elems) { @@ -3354,6 +3407,15 @@ static int cuda_attention_gemm(ffwd_cuda_ctx_t *ctx, int direct_kv_mode = bert_direct_kv_mode(); int use_bert_direct_kv = use_bert_flash && qkv_dtype != CUDA_DTYPE_F32 && qkv_dtype == out_dtype && direct_kv_mode >= 0; + // cuDNN fused attention takes exactly the shapes the built-in 16-bit flash + // kernels would run (same selection gates; FFWD_CUDA_SDPA=0 restores the + // hand-written kernels, and any cuDNN refusal falls back to them). It + // consumes the staged contiguous K/V, so the BERT direct-QKV read shortcut + // is bypassed while it is active. + int use_sdpa = sdpa_mode() >= 0 && ctx->sdpa && ctx->q16 && ctx->sdpa_batch_bucket > 0 && + (use_tc_kexp_bf16 || use_bert_flash); + if (use_sdpa) + use_bert_direct_kv = 0; int threads = 256; size_t exp_count = (size_t)total * q_dim; @@ -3406,6 +3468,47 @@ static int cuda_attention_gemm(ffwd_cuda_ctx_t *ctx, return -1; } + if (use_sdpa) { + // Q must match the K/V dtype: the Qwen path already wrote ctx->q16 + // inside the RoPE kernel; the BERT path packs it here from the + // (possibly still F32) fused QKV projection. + if (c->family == FFWD_FAMILY_BERT) { + size_t n = (size_t)total * q_dim; + int blocks = (int)((n + threads - 1) / threads); + if (out_dtype == CUDA_DTYPE_F16) { + if (qkv_dtype == CUDA_DTYPE_F16) + pack_q16_kernel<<<blocks, threads, 0, ctx->stream>>>( + (__half *)ctx->q16, (const __half *)qkv_src, total, q_dim, qkv_dim, q_offset); + else + pack_q16_kernel<<<blocks, threads, 0, ctx->stream>>>( + (__half *)ctx->q16, (const float *)qkv_src, total, q_dim, qkv_dim, q_offset); + } else { + if (qkv_dtype == CUDA_DTYPE_BF16) + pack_q16_kernel<<<blocks, threads, 0, ctx->stream>>>( + (__nv_bfloat16 *)ctx->q16, (const __nv_bfloat16 *)qkv_src, total, q_dim, + qkv_dim, q_offset); + else + pack_q16_kernel<<<blocks, threads, 0, ctx->stream>>>( + (__nv_bfloat16 *)ctx->q16, (const float *)qkv_src, total, q_dim, qkv_dim, + q_offset); + } + if (launch_check() != 0) + return -1; + } + const void *ksrc = out_dtype == CUDA_DTYPE_F16 ? ctx->kexp_f16 : ctx->kexp_bf16; + int bb = ctx->sdpa_batch_bucket; + const int32_t *seq = (const int32_t *)ctx->sdpa_meta; + const int32_t *rq = seq + bb; + const int32_t *rkv = rq + bb + 1; + int hkv_eff = ctx->sdpa_kv_row / hd; + if (ffwd_sdpa_run(ctx->sdpa, ctx->stream, ctx->q16, ksrc, ctx->vexp, out_16, seq, rq, rkv, bb, + ctx->sdpa_smax_bucket, H, hkv_eff, hd, scale, causal, + out_dtype == CUDA_DTYPE_F16) == 0) + return 0; + // Unsupported shape or execution failure: the staged K/V and Q remain + // valid for the built-in kernels below. + } + if (use_bert_flash) { int w = tc_kv_warps_or(8); #define BERT_FLASH_LAUNCH(HDV, W) \ @@ -3765,7 +3868,12 @@ static void bert_cuda_plan(ffwd_cuda_ctx_t *ctx, int total, bert_cuda_plan_t *p) int qkv16_mode = bert_qkv16_mode(); p->qkv16 = p->flash && p->cublaslt_avail && qkv16_mode >= 0 && (qkv16_mode > 0 || total >= BERT_QKV16_MIN_TOKENS); - p->direct_kv = p->flash && p->qkv16 && bert_direct_kv_mode() >= 0; + // The cuDNN fused-attention path reads staged contiguous K/V, so while it + // is a candidate the direct packed-QKV read is not planned and the K/V + // expansion buffers stay allocated (the dispatch relies on them, including + // for its fallback). + p->direct_kv = + p->flash && p->qkv16 && bert_direct_kv_mode() >= 0 && !(ctx->sdpa && sdpa_mode() >= 0); } static int ensure_buffers(ffwd_cuda_ctx_t *ctx, int total, int batch, int max_seq) { @@ -3793,6 +3901,10 @@ static int ensure_buffers(ffwd_cuda_ctx_t *ctx, int total, int batch, int max_se int need_act_f16 = ctx->weights_f16; int act_cols = 2 * c->intermediate_size; int ffn_gate_up_cols = 2 * c->intermediate_size; + // Packed 16-bit Q for the cuDNN fused-attention path: any shape whose + // 16-bit flash kernel could dispatch to cuDNN needs it staged. + int need_q16 = ctx->sdpa && sdpa_mode() >= 0 && + (is_bert ? bp.flash : (ctx->weights_bf16 && c->head_dim == 128)); if (is_bert) { need_x_norm = 0; @@ -3835,6 +3947,7 @@ static int ensure_buffers(ffwd_cuda_ctx_t *ctx, int total, int batch, int max_se scratch_key |= need_act_f16 << 16; scratch_key |= is_bert << 17; scratch_key |= (bp.flash ? 1 : 0) << 18; + scratch_key |= need_q16 << 19; if (total > ctx->seq_cap || scratch_key != ctx->scratch_key) { cudaFree(ctx->x); @@ -3853,6 +3966,7 @@ static int ensure_buffers(ffwd_cuda_ctx_t *ctx, int total, int batch, int max_se cudaFree(ctx->kexp_bf16); cudaFree(ctx->kexp_f16); cudaFree(ctx->vexp); + cudaFree(ctx->q16); cudaFree(ctx->act_bf16); cudaFree(ctx->act_f16); cudaFree(ctx->rope_cos); @@ -3870,6 +3984,7 @@ static int ensure_buffers(ffwd_cuda_ctx_t *ctx, int total, int batch, int max_se ctx->kexp = ctx->vexp = NULL; ctx->kexp_bf16 = NULL; ctx->kexp_f16 = NULL; + ctx->q16 = NULL; ctx->act_bf16 = NULL; ctx->act_f16 = NULL; ctx->rope_cos = NULL; @@ -3911,6 +4026,8 @@ static int ensure_buffers(ffwd_cuda_ctx_t *ctx, int total, int batch, int max_se CUDA_CHECK(cudaMalloc(&ctx->kexp_f16, (size_t)total * c->q_dim * sizeof(__half))); if (need_vexp) CUDA_CHECK(cudaMalloc((void **)&ctx->vexp, (size_t)total * c->q_dim * vexp_elem_bytes)); + if (need_q16) + CUDA_CHECK(cudaMalloc(&ctx->q16, (size_t)total * c->q_dim * sizeof(uint16_t))); if (need_act_bf16) CUDA_CHECK(cudaMalloc(&ctx->act_bf16, (size_t)total * act_cols * sizeof(__nv_bfloat16))); if (need_act_f16) @@ -4118,6 +4235,11 @@ static ffwd_cuda_ctx_t *cuda_ctx_from_model(ffwd_model_t *cpu, int own_cpu) { ffwd_cuda_free(ctx); return NULL; } + // cuDNN fused attention; NULL (missing/old libcudnn or FFWD_CUDA_SDPA=0) + // keeps the built-in kernels. Only 16-bit weight paths can dispatch to it, + // so exact-F32 models skip loading libcudnn entirely. + if (sdpa_mode() >= 0 && (ctx->weights_bf16 || ctx->weights_f16)) + ctx->sdpa = ffwd_sdpa_create(); // cuBLASLt handle + workspace + a small 16-bit bias scratch for the fused // bias/GeLU epilogue GEMMs (the bias is cast to the output dtype per call). ctx->lt_ws_bytes = (size_t)32 << 20; @@ -4213,6 +4335,10 @@ void ffwd_cuda_free(ffwd_cuda_ctx_t *ctx) { cudaFree(ctx->kexp_bf16); cudaFree(ctx->kexp_f16); cudaFree(ctx->vexp); + cudaFree(ctx->q16); + cudaFree(ctx->sdpa_meta); + free(ctx->sdpa_meta_host); + ffwd_sdpa_free(ctx->sdpa); cudaFree(ctx->act_bf16); cudaFree(ctx->act_f16); cudaFree(ctx->weight_f32); @@ -4243,6 +4369,60 @@ const ffwd_config_t *ffwd_cuda_config(const ffwd_cuda_ctx_t *ctx) { // Pack inputs into the device scratch every forward shares: token ids, the // per-sequence positions 0..L-1, and the batch offsets. Grows scratch via // ensure_buffers. Returns the packed token count in *out_total, or -1 on error. +// Stages the cuDNN fused-attention sequence metadata for one forward pass: +// per-sequence lengths plus ragged element offsets for Q/O and K/V rows, +// padded to a power-of-two batch bucket (padded entries are zero-length +// sequences) so the per-shape graph cache stays small. Layer-invariant, so it +// runs once per forward and every attention call reuses the device block. +static int cuda_sdpa_stage_meta(ffwd_cuda_ctx_t *ctx, const int *h_offsets, int batch) { + const ffwd_config_t *c = &ctx->config; + int total = h_offsets[batch]; + if ((long long)total * c->q_dim > INT_MAX) + return -1; // ragged offsets are int32 element counts + int max_L = 0; + for (int b = 0; b < batch; b++) { + int L = h_offsets[b + 1] - h_offsets[b]; + if (L > max_L) + max_L = L; + } + int bb = 1; + while (bb < batch) + bb <<= 1; + int smax = 64; + while (smax < max_L) + smax <<= 1; + int kv_row = (c->family == FFWD_FAMILY_QWEN3 && qwen_compact_kv()) ? c->n_kv_heads * c->head_dim + : c->q_dim; + int need = bb + 2 * (bb + 1); + if (need > ctx->sdpa_meta_cap) { + cudaFree(ctx->sdpa_meta); + free(ctx->sdpa_meta_host); + ctx->sdpa_meta = NULL; + ctx->sdpa_meta_cap = 0; + ctx->sdpa_meta_host = (int *)malloc((size_t)need * sizeof(int)); + if (!ctx->sdpa_meta_host) + return -1; + CUDA_CHECK(cudaMalloc((void **)&ctx->sdpa_meta, (size_t)need * sizeof(int))); + ctx->sdpa_meta_cap = need; + } + int *seq = ctx->sdpa_meta_host; + int *rq = seq + bb; + int *rkv = rq + bb + 1; + for (int b = 0; b < bb; b++) + seq[b] = b < batch ? h_offsets[b + 1] - h_offsets[b] : 0; + for (int b = 0; b <= bb; b++) { + int off = b <= batch ? h_offsets[b] : total; + rq[b] = off * c->q_dim; + rkv[b] = off * kv_row; + } + CUDA_CHECK(cudaMemcpyAsync(ctx->sdpa_meta, ctx->sdpa_meta_host, (size_t)need * sizeof(int), + cudaMemcpyHostToDevice, ctx->stream)); + ctx->sdpa_batch_bucket = bb; + ctx->sdpa_smax_bucket = smax; + ctx->sdpa_kv_row = kv_row; + return 0; +} + static int cuda_upload_packed_inputs(ffwd_cuda_ctx_t *ctx, const ffwd_input_t *inputs, int batch, @@ -4296,6 +4476,11 @@ static int cuda_upload_packed_inputs(ffwd_cuda_ctx_t *ctx, if (ce == cudaSuccess) ce = cudaMemcpyAsync(ctx->offsets, h_offsets, (size_t)(batch + 1) * sizeof(int), cudaMemcpyHostToDevice, ctx->stream); + // Metadata staging failure only disables the cuDNN attention path for this + // forward; the built-in kernels do not need it. + ctx->sdpa_batch_bucket = 0; + if (ce == cudaSuccess && ctx->q16) + (void)cuda_sdpa_stage_meta(ctx, h_offsets, batch); free(h_offsets); free(h_ids); free(h_pos); @@ -4842,9 +5027,9 @@ static int cuda_forward_batch(ffwd_cuda_ctx_t *ctx, const ffwd_input_t *inputs, } rms_norm_rope_qk_kernel<<<dim3(total, c->n_heads + c->n_kv_heads), 128, 0, ctx->stream>>>( - ctx->qkv, l->q_norm, l->k_norm, ctx->positions, ctx->rope_cos, ctx->rope_sin, total, - c->n_heads, c->n_kv_heads, c->head_dim, qkv_dim, q_offset, k_offset, c->qk_norm, - c->rms_norm_eps); + ctx->qkv, (__nv_bfloat16 *)ctx->q16, l->q_norm, l->k_norm, ctx->positions, ctx->rope_cos, + ctx->rope_sin, total, c->n_heads, c->n_kv_heads, c->head_dim, qkv_dim, q_offset, k_offset, + c->qk_norm, c->rms_norm_eps); if (launch_check() != 0) return -1; diff --git a/libffwd/cuda_sdpa.cu b/libffwd/cuda_sdpa.cu new file mode 100644 index 0000000..d3f7d09 --- /dev/null +++ b/libffwd/cuda_sdpa.cu @@ -0,0 +1,343 @@ +/* cuDNN graph-API fused attention (see cuda_sdpa.h for the contract). + * + * Compiled against the vendored cudnn-frontend headers with + * NV_CUDNN_FRONTEND_USE_DYNAMIC_LOADING: libcudnn is dlopen'd on first use + * instead of linked, so the CUDA build carries no hard cuDNN dependency. + * When the build has no cuDNN headers (FFWD_CUDA_SDPA undefined) this file + * compiles to stubs that report the feature as unavailable. + * + * One graph is built per attention shape class (batch bucket, sequence + * bucket, head counts, head dim, mask, dtype) and cached; variable lengths + * inside a bucket are handled by the kernel itself through per-sequence + * lengths and ragged offsets, so the cache stays small (buckets are powers + * of two). Shapes the installed cuDNN cannot fuse are negative-cached and + * the caller falls back to the built-in kernels. + */ + +#include "cuda_sdpa.h" + +#include <stdio.h> + +#ifndef FFWD_CUDA_SDPA + +extern "C" ffwd_sdpa_t *ffwd_sdpa_create(void) { return NULL; } +extern "C" void ffwd_sdpa_free(ffwd_sdpa_t *) {} +extern "C" int ffwd_sdpa_run(ffwd_sdpa_t *, + cudaStream_t, + const void *, + const void *, + const void *, + void *, + const int32_t *, + const int32_t *, + const int32_t *, + int, + int, + int, + int, + int, + float, + int, + int) { + return -2; +} + +#else /* FFWD_CUDA_SDPA */ + +# include <dlfcn.h> + +# include <cstdint> +# include <cstdlib> +# include <cstring> +# include <map> +# include <memory> +# include <tuple> +# include <unordered_map> + +# include <cudnn_frontend.h> + +namespace fe = cudnn_frontend; + +/* The dynamic-loading shim resolves every cuDNN symbol through this handle; + * the application owns its definition and initialization. */ +namespace cudnn_frontend { +void *cudnn_dlhandle = nullptr; +} + +namespace { + +/* Variant-pack tensor ids, fixed across all graphs. */ +enum : int64_t { + UID_Q = 1, + UID_K, + UID_V, + UID_O, + UID_SEQ_Q, + UID_SEQ_KV, + UID_RAGGED_Q, + UID_RAGGED_KV, + UID_RAGGED_O, +}; + +struct GraphKey { + int b, s, hq, hkv, d, causal, f16; + uint32_t scale_bits; + bool operator<(const GraphKey &o) const { + return std::tie(b, s, hq, hkv, d, causal, f16, scale_bits) < + std::tie(o.b, o.s, o.hq, o.hkv, o.d, o.causal, o.f16, o.scale_bits); + } +}; + +/* A built graph, or a negative entry (graph == nullptr) for shapes the + * installed cuDNN rejected. */ +struct GraphEntry { + std::shared_ptr<fe::graph::Graph> graph; + int64_t workspace_bytes = 0; +}; + +} // namespace + +struct ffwd_sdpa { + cudnnHandle_t handle = nullptr; + std::map<GraphKey, GraphEntry> cache; + void *workspace = nullptr; + size_t workspace_cap = 0; + int exec_error_reported = 0; + int unsupported_reported = 0; +}; + +/* Builds the fused SDPA graph for one shape class. Returns nullptr when the + * installed cuDNN does not support it (the caller negative-caches). */ +static std::shared_ptr<fe::graph::Graph> sdpa_build_graph(cudnnHandle_t handle, const GraphKey &k) { + auto graph = std::make_shared<fe::graph::Graph>(); + graph->set_io_data_type(k.f16 ? fe::DataType_t::HALF : fe::DataType_t::BFLOAT16) + .set_intermediate_data_type(fe::DataType_t::FLOAT) + .set_compute_data_type(fe::DataType_t::FLOAT); + + const int64_t b = k.b, hq = k.hq, hkv = k.hkv, d = k.d, s = k.s; + + /* Packed token-major ("THD") tensors: within one sequence the token + * stride is heads*d and the head stride is d; per-sequence start offsets + * come from the ragged-offset tensors, which make the nominal batch + * stride irrelevant for addressing (it still describes the padded + * bucket). Ragged offsets are element counts, int32, shape (b+1). */ + auto ragged_q = graph->tensor(fe::graph::Tensor_attributes() + .set_name("ragged_q") + .set_uid(UID_RAGGED_Q) + .set_dim({b + 1, 1, 1, 1}) + .set_stride({1, 1, 1, 1}) + .set_data_type(fe::DataType_t::INT32)); + auto ragged_kv = graph->tensor(fe::graph::Tensor_attributes() + .set_name("ragged_kv") + .set_uid(UID_RAGGED_KV) + .set_dim({b + 1, 1, 1, 1}) + .set_stride({1, 1, 1, 1}) + .set_data_type(fe::DataType_t::INT32)); + auto ragged_o = graph->tensor(fe::graph::Tensor_attributes() + .set_name("ragged_o") + .set_uid(UID_RAGGED_O) + .set_dim({b + 1, 1, 1, 1}) + .set_stride({1, 1, 1, 1}) + .set_data_type(fe::DataType_t::INT32)); + + auto Q = graph->tensor(fe::graph::Tensor_attributes() + .set_name("Q") + .set_uid(UID_Q) + .set_dim({b, hq, s, d}) + .set_stride({s * hq * d, d, hq * d, 1}) + .set_ragged_offset(ragged_q)); + auto K = graph->tensor(fe::graph::Tensor_attributes() + .set_name("K") + .set_uid(UID_K) + .set_dim({b, hkv, s, d}) + .set_stride({s * hkv * d, d, hkv * d, 1}) + .set_ragged_offset(ragged_kv)); + auto V = graph->tensor(fe::graph::Tensor_attributes() + .set_name("V") + .set_uid(UID_V) + .set_dim({b, hkv, s, d}) + .set_stride({s * hkv * d, d, hkv * d, 1}) + .set_ragged_offset(ragged_kv)); + + auto seq_q = graph->tensor(fe::graph::Tensor_attributes() + .set_name("seq_q") + .set_uid(UID_SEQ_Q) + .set_dim({b, 1, 1, 1}) + .set_stride({1, 1, 1, 1}) + .set_data_type(fe::DataType_t::INT32)); + auto seq_kv = graph->tensor(fe::graph::Tensor_attributes() + .set_name("seq_kv") + .set_uid(UID_SEQ_KV) + .set_dim({b, 1, 1, 1}) + .set_stride({1, 1, 1, 1}) + .set_data_type(fe::DataType_t::INT32)); + + float scale; + std::memcpy(&scale, &k.scale_bits, sizeof(scale)); + auto opts = fe::graph::SDPA_attributes() + .set_name("ffwd_sdpa") + .set_generate_stats(false) + .set_attn_scale(scale) + .set_padding_mask(true) + .set_seq_len_q(seq_q) + .set_seq_len_kv(seq_kv); + if (k.causal) + opts.set_causal_mask(true); + + /* Ragged inputs are only expressible by the flash engines (the classic + * multi-op graph on SM80+, the unified fused op on newer cuDNN), so an + * unfusable shape fails the build here instead of silently running a + * materialized fallback. Known gates: GQA+ragged needs cuDNN >= 9.6, + * ragged on Ampere/Ada (sm8x) needs >= 9.18. */ + auto [O, stats] = graph->sdpa(Q, K, V, opts); + (void)stats; + O->set_output(true) + .set_uid(UID_O) + .set_dim({b, hq, s, d}) + .set_stride({s * hq * d, d, hq * d, 1}) + .set_ragged_offset(ragged_o); + + if (!graph->build(handle, {fe::HeurMode_t::A}).is_good()) + return nullptr; + return graph; +} + +extern "C" ffwd_sdpa_t *ffwd_sdpa_create(void) { + const char *lib = getenv("FFWD_CUDNN_LIB"); + void *dl = nullptr; + if (lib && lib[0]) + dl = dlopen(lib, RTLD_NOW | RTLD_LOCAL); + if (!dl) + dl = dlopen("libcudnn.so.9", RTLD_NOW | RTLD_LOCAL); + if (!dl) + dl = dlopen("libcudnn.so", RTLD_NOW | RTLD_LOCAL); + if (!dl) { + fprintf(stderr, "ffwd-cuda: libcudnn not found, using built-in attention kernels\n"); + return NULL; + } + cudnn_frontend::cudnn_dlhandle = dl; + + ffwd_sdpa_t *s = NULL; + try { + /* The fused SDPA support surface used here (ragged THD, GQA head + * counts, padding mask) is stable from cuDNN 9 on. */ + if (fe::detail::get_backend_version() < 90000) { + fprintf(stderr, "ffwd-cuda: cuDNN %s too old (need >= 9.0), using built-in kernels\n", + fe::detail::get_backend_version_string().c_str()); + return NULL; + } + s = new ffwd_sdpa(); + if (fe::detail::create_handle(&s->handle) != CUDNN_STATUS_SUCCESS) { + delete s; + return NULL; + } + } catch (...) { + delete s; + return NULL; + } + return s; +} + +extern "C" void ffwd_sdpa_free(ffwd_sdpa_t *s) { + if (!s) + return; + s->cache.clear(); + if (s->workspace) + cudaFree(s->workspace); + if (s->handle) + fe::detail::destroy_handle(s->handle); + delete s; +} + +extern "C" int ffwd_sdpa_run(ffwd_sdpa_t *s, + cudaStream_t stream, + const void *q, + const void *k, + const void *v, + void *o, + const int32_t *seq_len, + const int32_t *ragged_q, + const int32_t *ragged_kv, + int batch, + int s_max, + int hq, + int hkv, + int d, + float scale, + int causal, + int is_f16) { + if (!s) + return -2; + GraphKey key; + key.b = batch; + key.s = s_max; + key.hq = hq; + key.hkv = hkv; + key.d = d; + key.causal = causal; + key.f16 = is_f16; + std::memcpy(&key.scale_bits, &scale, sizeof(scale)); + + try { + auto it = s->cache.find(key); + if (it == s->cache.end()) { + GraphEntry e; + e.graph = sdpa_build_graph(s->handle, key); + if (e.graph && !e.graph->get_workspace_size(e.workspace_bytes).is_good()) + e.graph = nullptr; + if (!e.graph && !s->unsupported_reported) { + s->unsupported_reported = 1; + fprintf(stderr, + "ffwd-cuda: cuDNN %s cannot fuse this attention shape " + "(hq=%d hkv=%d d=%d causal=%d), using built-in kernels\n", + fe::detail::get_backend_version_string().c_str(), hq, hkv, d, causal); + } + it = s->cache.emplace(key, std::move(e)).first; + } + if (!it->second.graph) + return -2; + + if ((size_t)it->second.workspace_bytes > s->workspace_cap) { + if (s->workspace) + cudaFree(s->workspace); + s->workspace = NULL; + s->workspace_cap = 0; + if (cudaMalloc(&s->workspace, (size_t)it->second.workspace_bytes) != cudaSuccess) + return -1; + s->workspace_cap = (size_t)it->second.workspace_bytes; + } + + if (fe::detail::set_stream(s->handle, stream) != CUDNN_STATUS_SUCCESS) + return -1; + + std::unordered_map<int64_t, void *> pack = { + {UID_Q, const_cast<void *>(q)}, + {UID_K, const_cast<void *>(k)}, + {UID_V, const_cast<void *>(v)}, + {UID_O, o}, + {UID_SEQ_Q, const_cast<int32_t *>(seq_len)}, + {UID_SEQ_KV, const_cast<int32_t *>(seq_len)}, + {UID_RAGGED_Q, const_cast<int32_t *>(ragged_q)}, + {UID_RAGGED_KV, const_cast<int32_t *>(ragged_kv)}, + {UID_RAGGED_O, const_cast<int32_t *>(ragged_q)}, + }; + if (!it->second.graph->execute(s->handle, pack, s->workspace).is_good()) { + if (!s->exec_error_reported) { + s->exec_error_reported = 1; + fprintf(stderr, "ffwd-cuda: cuDNN SDPA execution failed, using built-in kernels\n"); + } + return -1; + } + } catch (const std::exception &ex) { + if (!s->exec_error_reported) { + s->exec_error_reported = 1; + fprintf(stderr, "ffwd-cuda: cuDNN SDPA error: %s\n", ex.what()); + } + return -1; + } catch (...) { + return -1; + } + return 0; +} + +#endif /* FFWD_CUDA_SDPA */ diff --git a/libffwd/cuda_sdpa.h b/libffwd/cuda_sdpa.h new file mode 100644 index 0000000..a6ee80b --- /dev/null +++ b/libffwd/cuda_sdpa.h @@ -0,0 +1,69 @@ +/* cuDNN-backed fused scaled-dot-product attention for the CUDA backend. + * + * Wraps the cuDNN graph API (via the vendored cudnn-frontend headers) behind a + * small C interface so cuda.cu can dispatch packed variable-length attention + * to NVIDIA's fused flash kernels. libcudnn is dlopen'd at runtime: when it is + * absent or too old, ffwd_sdpa_create returns NULL and the caller keeps the + * built-in kernels. Builds without cuDNN headers compile this interface to + * stubs (see cuda_sdpa.cu). + */ +#ifndef FFWD_CUDA_SDPA_H +#define FFWD_CUDA_SDPA_H + +#include <cuda_runtime.h> +#include <stdint.h> + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct ffwd_sdpa ffwd_sdpa_t; + +/* Returns NULL when fused SDPA is unavailable (built without cuDNN headers, + * libcudnn missing at runtime, or an unsupported cuDNN/driver combination). */ +ffwd_sdpa_t *ffwd_sdpa_create(void); +void ffwd_sdpa_free(ffwd_sdpa_t *s); + +/* Fused SDPA forward over a packed variable-length batch. + * + * Layout (all device pointers, 16-bit elements; bf16 when is_f16 == 0, f16 + * otherwise): + * q, o: [total_tokens, hq * d], token-major, head then dim within a token + * k, v: [total_tokens, hkv * d] (compact GQA: one row per kv head) + * Softmax and accumulation run in f32 inside the fused kernel. + * + * Sequence metadata (device int32, prepared by the caller): + * seq_len: [batch] per-sequence token counts (0 for padded entries) + * ragged_q: [batch + 1] prefix offsets in ELEMENTS of q/o rows + * (offsets[b] * hq * d), last entry = total * hq * d + * ragged_kv: [batch + 1] same for k/v rows (offsets[b] * hkv * d) + * batch and s_max are the graph dimensions and may exceed the live batch / + * longest sequence (bucketing): padded batch entries carry seq_len 0 and a + * repeated final ragged offset. + * + * Returns 0 on success; -1 on an execution error; -2 when this shape is + * unsupported by the installed cuDNN (negative-cached, cheap to re-ask). + * On nonzero the caller must fall back to the built-in kernels. */ +int ffwd_sdpa_run(ffwd_sdpa_t *s, + cudaStream_t stream, + const void *q, + const void *k, + const void *v, + void *o, + const int32_t *seq_len, + const int32_t *ragged_q, + const int32_t *ragged_kv, + int batch, + int s_max, + int hq, + int hkv, + int d, + float scale, + int causal, + int is_f16); + +#ifdef __cplusplus +} +#endif + +#endif /* FFWD_CUDA_SDPA_H */ |
