aboutsummaryrefslogtreecommitdiff
path: root/deps/cudnn-frontend/include/cudnn_frontend/node
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-07-02 19:44:08 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-07-02 19:44:08 +0200
commit6a10d3577232f1fd646a20f22a4a1ab4ec29f28a (patch)
treee4a077b064876ddb39144935fa256a4ab0692488 /deps/cudnn-frontend/include/cudnn_frontend/node
parent98a713e86c23bc6098bf7a5558507a9f27a7a1c1 (diff)
downloadffwd-6a10d3577232f1fd646a20f22a4a1ab4ec29f28a.tar.gz
ffwd-6a10d3577232f1fd646a20f22a4a1ab4ec29f28a.zip
cuda: add cuDNN fused attention path with runtime fallback
Diffstat (limited to 'deps/cudnn-frontend/include/cudnn_frontend/node')
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/node/adaptive_layernorm.h454
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/node/batchnorm.h268
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/node/batchnorm_inference.h156
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/node/block_scale_dequantize.h169
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/node/block_scale_quantize.h205
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/node/bn_finalize.h263
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/node/concatenate.h163
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/node/conv_dgrad.h205
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/node/conv_fprop.h254
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/node/conv_wgrad.h201
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/node/dbn.h206
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/node/dbn_weight.h215
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/node/diagonal_band_mask.h474
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/node/dln.h226
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/node/genstats.h147
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/node/instancenorm.h414
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/node/layernorm.h259
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/node/matmul.h253
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/node/matmul_fp8.h104
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/node/moe_grouped_matmul.h221
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/node/moe_grouped_matmul_bwd.h181
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/node/paged_cache_load.h153
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/node/pointwise.h390
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/node/reduction.h232
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/node/resample.h291
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/node/reshape.h156
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/node/rmsnorm.h406
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/node/rng.h187
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/node/rope.h146
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/node/rope_backward.h143
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/node/scaled_dot_product_flash_attention.h2730
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/node/sdpa_fp8_bwd.h1172
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/node/sdpa_support_surface.h553
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/node/slice.h224
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/node/softmax.h366
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/node/transpose.h164
36 files changed, 12351 insertions, 0 deletions
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/adaptive_layernorm.h b/deps/cudnn-frontend/include/cudnn_frontend/node/adaptive_layernorm.h
new file mode 100644
index 0000000..86eaa6d
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/node/adaptive_layernorm.h
@@ -0,0 +1,454 @@
+#pragma once
+
+#include "../../cudnn_frontend_Heuristics.h"
+#include "../../cudnn_frontend_Logging.h"
+
+#include "../graph_helpers.h"
+#include "../node_interface.h"
+
+namespace cudnn_frontend {
+
+namespace graph {
+class AdaLayerNormNode : public NodeCRTP<AdaLayerNormNode> {
+ public:
+ AdaLayernorm_attributes attributes;
+
+ AdaLayerNormNode(AdaLayernorm_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP(context), attributes(std::move(attributes_)) {}
+
+ Type
+ getType() override final {
+ return Type::ADALAYERNORM;
+ }
+
+ error_t
+ infer_properties_node() override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for adalayernorm node " << attributes.name);
+
+ attributes.fill_from_context(context);
+
+ auto X = attributes.inputs[AdaLayernorm_attributes::input_names::X];
+ auto Y = attributes.outputs[AdaLayernorm_attributes::output_names::Y];
+
+ // Only infer dims and strides if user did not set them
+ if (Y->get_dim().empty()) {
+ Y->set_dim(X->get_dim());
+ }
+ if (Y->get_stride().empty()) {
+ Y->set_stride(X->get_stride());
+ }
+
+ // scale_bias dim is 1,c,h,w
+ auto scale_bias_dim = X->get_dim();
+ scale_bias_dim[0] = 1;
+
+ auto scale = attributes.inputs[AdaLayernorm_attributes::input_names::SCALE];
+ // Only infer dims and strides if user did not set them
+ if (scale->get_dim().empty()) {
+ scale->set_dim(scale_bias_dim);
+ }
+ if (scale->get_stride().empty()) {
+ auto const& scale_dim = scale->get_dim();
+ std::vector<int64_t> stride_order;
+ CHECK_CUDNN_FRONTEND_ERROR(
+ detail::generate_stride_order_preserving_format(X->get_stride(), scale_dim.size(), stride_order));
+ scale->set_stride(detail::generate_stride(scale_dim, stride_order));
+ }
+
+ auto bias = attributes.inputs[AdaLayernorm_attributes::input_names::BIAS];
+ // Only infer dims and strides if user did not set them
+ if (bias->get_dim().empty()) {
+ bias->set_dim(scale_bias_dim);
+ }
+ if (bias->get_stride().empty()) {
+ auto const& bias_dim = bias->get_dim();
+ std::vector<int64_t> stride_order;
+ CHECK_CUDNN_FRONTEND_ERROR(
+ detail::generate_stride_order_preserving_format(X->get_stride(), bias_dim.size(), stride_order));
+ bias->set_stride(detail::generate_stride(bias_dim, stride_order));
+ }
+
+ if (attributes.forward_phase == NormFwdPhase_t::TRAINING) {
+ // stats dim is x where scale == 1 else 1
+ auto stats_dim = X->get_dim();
+ for (size_t i = 1; i < stats_dim.size(); i++) {
+ if (scale->get_dim()[i] != 1) {
+ stats_dim[i] = 1;
+ }
+ }
+
+ auto mean = attributes.outputs[AdaLayernorm_attributes::output_names::MEAN];
+ // Only infer dims and strides if user did not set them
+ if (mean->get_dim().empty()) {
+ mean->set_dim(stats_dim);
+ }
+ if (mean->get_stride().empty()) {
+ auto const& mean_dim = mean->get_dim();
+ std::vector<int64_t> stride_order;
+ CHECK_CUDNN_FRONTEND_ERROR(
+ detail::generate_stride_order_preserving_format(X->get_stride(), mean_dim.size(), stride_order));
+ mean->set_stride(detail::generate_stride(mean_dim, stride_order));
+ }
+
+ auto inv_var = attributes.outputs[AdaLayernorm_attributes::output_names::INV_VARIANCE];
+ // Only infer dims and strides if user did not set them
+ if (inv_var->get_dim().empty()) {
+ inv_var->set_dim(stats_dim);
+ }
+ if (inv_var->get_stride().empty()) {
+ auto const& inv_var_dim = inv_var->get_dim();
+ std::vector<int64_t> stride_order;
+ CHECK_CUDNN_FRONTEND_ERROR(
+ detail::generate_stride_order_preserving_format(X->get_stride(), inv_var_dim.size(), stride_order));
+ inv_var->set_stride(detail::generate_stride(inv_var_dim, stride_order));
+ }
+ }
+
+ // Set scalar tensors
+ std::vector<int64_t> ones(X->get_dim().size(), 1);
+ auto infer_scalar_tensors = [&ones](std::shared_ptr<Tensor_attributes>& T) {
+ // Only infer dims and strides if user did not set them
+ if (T->get_dim().empty()) {
+ T->set_dim(ones);
+ }
+ if (T->get_stride().empty()) {
+ T->set_stride(ones);
+ }
+ };
+ infer_scalar_tensors(attributes.inputs[AdaLayernorm_attributes::input_names::EPSILON]);
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ pre_validate_node() const override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: " << "Validating AdaLayerNormNode " << attributes.name);
+ // Norm forward phase should be set
+ RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.forward_phase == NormFwdPhase_t::NOT_SET,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "Forward phase not set of adalayernorm node.");
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ create_cudnn_operations(
+ std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations,
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations,
+ managed_backend_descriptor_t& raw_operations,
+ std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final {
+ getLogger() << "[cudnn_frontend] INFO: " << "Building AdaLayernorm operations " << attributes.name << std::endl;
+
+ auto cudnn_ver_error = error_t{error_code_t::GRAPH_NOT_SUPPORTED, "AdaLN fwd requires cuDNN v9.9.0"};
+#if (CUDNN_VERSION >= 90900)
+ NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(90900, cudnn_ver_error);
+ CUDNN_FRONTEND_UNUSED(operations);
+ auto adalayernorm_operation =
+ make_shared_backend_pointer((cudnnBackendDescriptorType_t)CUDNN_BACKEND_OPERATION_NORM_FORWARD_DESCRIPTOR);
+
+ cudnnBackendNormMode_t cudnn_norm_mode;
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormMode_t::ADA_LAYER_NORM, cudnn_norm_mode));
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_MODE,
+ CUDNN_TYPE_NORM_MODE,
+ 1,
+ &cudnn_norm_mode));
+
+ cudnnBackendNormFwdPhase_t cudnn_norm_fwd_phase;
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.forward_phase, cudnn_norm_fwd_phase));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_PHASE,
+ CUDNN_TYPE_NORM_FWD_PHASE,
+ 1,
+ &cudnn_norm_fwd_phase));
+
+ auto X = attributes.inputs.find(AdaLayernorm_attributes::input_names::X)->second;
+ auto backend_x = tensors[X->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_XDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_x));
+
+ auto Scale = attributes.inputs.find(AdaLayernorm_attributes::input_names::SCALE)->second;
+ auto backend_scale = tensors[Scale->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_SCALE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_scale));
+
+ auto Bias_iter = attributes.inputs.find(AdaLayernorm_attributes::input_names::BIAS);
+ if (Bias_iter != attributes.inputs.end() && Bias_iter->second->get_is_virtual() == false) {
+ auto backend_bias = tensors[Bias_iter->second->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_BIAS_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_bias));
+ }
+
+ auto Epsilon = attributes.inputs.find(AdaLayernorm_attributes::input_names::EPSILON)->second;
+ auto backend_epsilon = tensors[Epsilon->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_EPSILON_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_epsilon));
+
+ auto Y = attributes.outputs.find(AdaLayernorm_attributes::output_names::Y)->second;
+ auto backend_y = tensors[Y->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_YDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_y));
+
+ if (attributes.forward_phase == NormFwdPhase_t::TRAINING) {
+ auto Mean = attributes.outputs.find(AdaLayernorm_attributes::output_names::MEAN)->second;
+ auto backend_mean = tensors[Mean->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_MEAN_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_mean));
+
+ auto Inv_variance = attributes.outputs.find(AdaLayernorm_attributes::output_names::INV_VARIANCE)->second;
+ auto backend_inv_variance = tensors[Inv_variance->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_INV_VARIANCE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_inv_variance));
+ }
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(adalayernorm_operation->get_backend_descriptor()));
+
+ raw_operations.push_back(adalayernorm_operation);
+
+ auto const& non_virtual_uids = attributes.get_non_virtual_uids();
+ uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end());
+ return {error_code_t::OK, ""};
+#else
+ CUDNN_FRONTEND_UNUSED(uids_involved_in_operations);
+ CUDNN_FRONTEND_UNUSED(operations);
+ CUDNN_FRONTEND_UNUSED(raw_operations);
+ CUDNN_FRONTEND_UNUSED(tensors);
+ return cudnn_ver_error;
+#endif
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ j.update(R"( {"tag": "ADA_LAYER_NORM"})"_json);
+ }
+#endif
+};
+
+/*******/
+
+class DAdaLayerNormNode : public NodeCRTP<DAdaLayerNormNode> {
+ public:
+ AdaLayernorm_backward_attributes attributes;
+
+ DAdaLayerNormNode(AdaLayernorm_backward_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP(context), attributes(std::move(attributes_)) {}
+
+ Type
+ getType() override final {
+ return Type::DADALAYERNORM;
+ }
+
+ error_t
+ infer_properties_node() override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for DAdaLayerNorm node " << attributes.name);
+
+ attributes.fill_from_context(context);
+
+ // TODO: Only inferencing from X works today.
+ auto X = attributes.inputs[AdaLayernorm_backward_attributes::input_names::X];
+ auto const x_tensor_dim = X->get_dim();
+
+ auto DY = attributes.inputs[AdaLayernorm_backward_attributes::input_names::DY];
+ auto dy_tensor_dim = DY->get_dim();
+
+ // Only infer dims and strides if user did not set them
+ if (dy_tensor_dim.empty()) {
+ dy_tensor_dim.resize(x_tensor_dim.size());
+ DY->set_dim(x_tensor_dim);
+ }
+ if (DY->get_stride().empty()) {
+ auto const& DY_dim = DY->get_dim();
+ // Default to NCHW
+ auto const& stride_order = detail::generate_row_major_stride_order(DY_dim.size());
+ DY->set_stride(detail::generate_stride(DY_dim, stride_order));
+ }
+
+ auto DX = attributes.outputs[AdaLayernorm_backward_attributes::output_names::DX];
+ auto dx_tensor_dim = DX->get_dim();
+ // Only infer dims and strides if user did not set them
+ if (dx_tensor_dim.empty()) {
+ dx_tensor_dim.resize(x_tensor_dim.size());
+ DX->set_dim(x_tensor_dim);
+ }
+ if (DX->get_stride().empty()) {
+ auto const& DX_dim = DX->get_dim();
+ // Default to NCHW
+ auto const& stride_order = detail::generate_row_major_stride_order(DX_dim.size());
+ DX->set_stride(detail::generate_stride(DX_dim, stride_order));
+ }
+
+ auto SCALE = attributes.inputs[AdaLayernorm_backward_attributes::input_names::SCALE];
+ auto scale_bias_dim = SCALE->get_dim();
+
+ // Set channel length tensors
+ auto infer_scale_bias_tensors = [&scale_bias_dim](std::shared_ptr<Tensor_attributes>& T) {
+ auto tensor_dim = T->get_dim();
+ // Only infer dims and strides if user did not set them
+ if (tensor_dim.empty()) {
+ T->set_dim(scale_bias_dim);
+ }
+ if (T->get_stride().empty()) {
+ auto const& T_dim = T->get_dim();
+ // Default to NCHW
+ auto const& stride_order = detail::generate_row_major_stride_order(T_dim.size());
+ T->set_stride(detail::generate_stride(T_dim, stride_order));
+ }
+ };
+
+ infer_scale_bias_tensors(attributes.outputs[AdaLayernorm_backward_attributes::output_names::DSCALE]);
+ auto DBIAS = attributes.outputs.at(AdaLayernorm_backward_attributes::output_names::DBIAS);
+ if (DBIAS->get_is_virtual() == false) {
+ infer_scale_bias_tensors(DBIAS);
+ }
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ pre_validate_node() const override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating DAdaLayerNormNode node " << attributes.name);
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ create_cudnn_operations(
+ std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations,
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations,
+ managed_backend_descriptor_t& raw_operations,
+ std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final {
+ getLogger() << "[cudnn_frontend] INFO: " << "Building DAdaLayerNormNode operations " << attributes.name
+ << std::endl;
+ auto cudnn_ver_error = error_t{error_code_t::GRAPH_NOT_SUPPORTED, "AdaLN bwd requires cuDNN v9.9.0"};
+#if (CUDNN_VERSION >= 90900)
+ NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(90900, cudnn_ver_error);
+ CUDNN_FRONTEND_UNUSED(operations);
+ auto adalayernorm_operation =
+ make_shared_backend_pointer((cudnnBackendDescriptorType_t)CUDNN_BACKEND_OPERATION_NORM_BACKWARD_DESCRIPTOR);
+
+ cudnnBackendNormMode_t cudnn_norm_mode;
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormMode_t::ADA_LAYER_NORM, cudnn_norm_mode));
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_MODE,
+ CUDNN_TYPE_NORM_MODE,
+ 1,
+ &cudnn_norm_mode));
+
+ auto X = attributes.inputs.find(AdaLayernorm_backward_attributes::input_names::X)->second;
+ auto backend_x = tensors[X->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_XDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_x));
+
+ auto Mean = attributes.inputs.find(AdaLayernorm_backward_attributes::input_names::MEAN)->second;
+ auto backend_mean = tensors[Mean->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_MEAN_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_mean));
+
+ auto Inv_variance = attributes.inputs.find(AdaLayernorm_backward_attributes::input_names::INV_VARIANCE)->second;
+ auto backend_inv_variance = tensors[Inv_variance->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_INV_VARIANCE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_inv_variance));
+
+ auto Dy = attributes.inputs.find(AdaLayernorm_backward_attributes::input_names::DY)->second;
+ auto backend_dy = tensors[Dy->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_DYDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_dy));
+
+ auto Scale = attributes.inputs.find(AdaLayernorm_backward_attributes::input_names::SCALE)->second;
+ auto backend_scale = tensors[Scale->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_SCALE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_scale));
+
+ auto Dx = attributes.outputs.find(AdaLayernorm_backward_attributes::output_names::DX)->second;
+ auto backend_dx = tensors[Dx->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_DXDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_dx));
+
+ auto Dscale = attributes.outputs.find(AdaLayernorm_backward_attributes::output_names::DSCALE)->second;
+ auto backend_dscale = tensors[Dscale->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_DSCALE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_dscale));
+
+ auto Dbias_iter = attributes.outputs.find(AdaLayernorm_backward_attributes::output_names::DBIAS);
+ if (Dbias_iter != attributes.outputs.end() && Dbias_iter->second->get_is_virtual() == false) {
+ auto backend_dbias = tensors[Dbias_iter->second->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_DBIAS_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_dbias));
+ }
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(adalayernorm_operation->get_backend_descriptor()));
+
+ raw_operations.push_back(adalayernorm_operation);
+
+ auto const& non_virtual_uids = attributes.get_non_virtual_uids();
+ uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end());
+ return {error_code_t::OK, ""};
+#else
+ CUDNN_FRONTEND_UNUSED(uids_involved_in_operations);
+ CUDNN_FRONTEND_UNUSED(operations);
+ CUDNN_FRONTEND_UNUSED(raw_operations);
+ CUDNN_FRONTEND_UNUSED(tensors);
+ return cudnn_ver_error;
+#endif
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ j.update(R"( {"tag": "ADA_LAYER_NORM_BPROP"})"_json);
+ }
+#endif
+};
+
+} // namespace graph
+
+} // namespace cudnn_frontend \ No newline at end of file
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/batchnorm.h b/deps/cudnn-frontend/include/cudnn_frontend/node/batchnorm.h
new file mode 100644
index 0000000..69d232b
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/node/batchnorm.h
@@ -0,0 +1,268 @@
+#pragma once
+
+#include "../graph_helpers.h"
+#include "../node_interface.h"
+
+namespace cudnn_frontend {
+
+namespace graph {
+class BatchNormNode : public NodeCRTP<BatchNormNode> {
+ public:
+ Batchnorm_attributes attributes;
+
+ BatchNormNode(Batchnorm_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP(context), attributes(std::move(attributes_)) {}
+
+ Type
+ getType() override final {
+ return Type::BATCHNORM;
+ }
+
+ error_t
+ infer_properties_node() override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for batchnorm node " << attributes.name);
+
+ attributes.fill_from_context(context);
+
+ auto X = attributes.inputs[Batchnorm_attributes::input_names::X];
+ auto Y = attributes.outputs[Batchnorm_attributes::output_names::Y];
+ // Only infer dims and strides if user did not set them
+ if (Y->get_dim().empty()) {
+ Y->set_dim(X->get_dim());
+ }
+ if (Y->get_stride().empty()) {
+ Y->set_stride(X->get_stride());
+ }
+
+ // Set channel length tensors
+ auto const x_tensor_dim = X->get_dim();
+ auto infer_per_channel_tensors = [&x_tensor_dim](std::shared_ptr<Tensor_attributes>& T) {
+ auto tensor_dim = T->get_dim();
+ // Only infer dims and strides if user did not set them
+ if (tensor_dim.empty()) {
+ tensor_dim.resize(x_tensor_dim.size(), 1);
+ tensor_dim[1] = x_tensor_dim[1];
+ T->set_dim(tensor_dim);
+ }
+ if (T->get_stride().empty()) {
+ auto const& T_dim = T->get_dim();
+ // Default to NHWC
+ auto const& stride_order = detail::generate_NHWC_stride_order(T_dim.size());
+ T->set_stride(detail::generate_stride(T_dim, stride_order));
+ }
+ };
+ infer_per_channel_tensors(attributes.outputs[Batchnorm_attributes::output_names::MEAN]);
+ infer_per_channel_tensors(attributes.outputs[Batchnorm_attributes::output_names::INV_VARIANCE]);
+
+ auto has_running_stats = attributes.inputs[Batchnorm_attributes::input_names::PREV_RUNNING_MEAN] ||
+ attributes.inputs[Batchnorm_attributes::input_names::PREV_RUNNING_VAR];
+
+ if (has_running_stats) {
+ infer_per_channel_tensors(attributes.outputs[Batchnorm_attributes::output_names::NEXT_RUNNING_MEAN]);
+ infer_per_channel_tensors(attributes.outputs[Batchnorm_attributes::output_names::NEXT_RUNNING_VAR]);
+ }
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ create_cudnn_operations(
+ std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations,
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations,
+ managed_backend_descriptor_t& raw_operations,
+ std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final {
+ CUDNN_FRONTEND_UNUSED(raw_operations);
+ CUDNN_FE_LOG_LABEL("INFO: Building BatchNormNode operations " << attributes.name << " ");
+
+ // Create operation by directly calling cuDNN backend API
+ Operation_v8 batchnorm_operation;
+
+ _CUDNN_CHECK_CUDNN_ERROR(
+ batchnorm_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_NORM_FORWARD_DESCRIPTOR));
+
+ // Set norm mode to BATCH_NORM
+ cudnnBackendNormMode_t cudnn_norm_mode;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormMode_t::BATCH_NORM, cudnn_norm_mode));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_MODE,
+ CUDNN_TYPE_NORM_MODE,
+ 1,
+ &cudnn_norm_mode));
+
+ // Set forward phase to TRAINING
+ cudnnBackendNormFwdPhase_t cudnn_norm_fwd_phase;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormFwdPhase_t::TRAINING, cudnn_norm_fwd_phase));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_PHASE,
+ CUDNN_TYPE_NORM_FWD_PHASE,
+ 1,
+ &cudnn_norm_fwd_phase));
+
+ // Set input tensor X
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Batchnorm_attributes::input_names::X);
+ auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_XDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &x_desc));
+
+ // Set saved mean and inv_variance
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(MEAN, Batchnorm_attributes::output_names::MEAN);
+ auto mean_desc = tensors.at(MEAN->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_MEAN_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &mean_desc));
+
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(INV_VARIANCE, Batchnorm_attributes::output_names::INV_VARIANCE);
+ auto inv_var_desc = tensors.at(INV_VARIANCE->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_INV_VARIANCE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &inv_var_desc));
+
+ // Set scale and bias tensors
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SCALE, Batchnorm_attributes::input_names::SCALE);
+ auto scale_desc = tensors.at(SCALE->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_SCALE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &scale_desc));
+
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(BIAS, Batchnorm_attributes::input_names::BIAS);
+ auto bias_desc = tensors.at(BIAS->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_BIAS_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &bias_desc));
+
+ // Set epsilon tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(EPSILON, Batchnorm_attributes::input_names::EPSILON);
+ auto epsilon_desc = tensors.at(EPSILON->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_EPSILON_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &epsilon_desc));
+
+ // Check for running stats
+ bool has_running_stats = true;
+ auto it = attributes.inputs.find(Batchnorm_attributes::input_names::PREV_RUNNING_MEAN);
+ if (it == attributes.inputs.end() || it->second == nullptr) {
+ has_running_stats = false;
+ }
+
+ if (has_running_stats) {
+ // Set momentum (exp decay factor)
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(MOMENTUM, Batchnorm_attributes::input_names::MOMENTUM);
+ auto momentum_desc = tensors.at(MOMENTUM->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_EXP_AVG_FACTOR_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &momentum_desc));
+
+ // Set prev running mean and var
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(PREV_RUNNING_MEAN,
+ Batchnorm_attributes::input_names::PREV_RUNNING_MEAN);
+ auto prev_mean_desc = tensors.at(PREV_RUNNING_MEAN->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_INPUT_RUNNING_MEAN_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &prev_mean_desc));
+
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(PREV_RUNNING_VAR,
+ Batchnorm_attributes::input_names::PREV_RUNNING_VAR);
+ auto prev_var_desc = tensors.at(PREV_RUNNING_VAR->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_INPUT_RUNNING_VAR_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &prev_var_desc));
+
+ // Set next running mean and var
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(NEXT_RUNNING_MEAN,
+ Batchnorm_attributes::output_names::NEXT_RUNNING_MEAN);
+ auto next_mean_desc = tensors.at(NEXT_RUNNING_MEAN->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_OUTPUT_RUNNING_MEAN_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &next_mean_desc));
+
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(NEXT_RUNNING_VAR,
+ Batchnorm_attributes::output_names::NEXT_RUNNING_VAR);
+ auto next_var_desc = tensors.at(NEXT_RUNNING_VAR->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_OUTPUT_RUNNING_VAR_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &next_var_desc));
+ }
+
+ // Set output tensor Y
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(Y, Batchnorm_attributes::output_names::Y);
+ auto y_desc = tensors.at(Y->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_YDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &y_desc));
+
+ // Set peer stat tensors if any
+ if (!attributes.peer_stats.empty()) {
+ std::vector<cudnnBackendDescriptor_t> peer_stat_descs;
+ for (auto const& peer_stat : attributes.peer_stats) {
+ peer_stat_descs.push_back(tensors.at(peer_stat->get_uid())->get_raw_desc());
+ }
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_PEER_STAT_DESCS,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ peer_stat_descs.size(),
+ peer_stat_descs.data()));
+ }
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(batchnorm_operation.get_raw_desc()));
+
+ operations.push_back(std::make_shared<Operation_v8>(std::move(batchnorm_operation)));
+
+ auto const& non_virtual_uids = attributes.get_non_virtual_uids();
+ uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end());
+ return {error_code_t::OK, ""};
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ j.update(R"( {"tag": "BATCHNORM"})"_json);
+ }
+#endif
+};
+
+} // namespace graph
+
+} // namespace cudnn_frontend \ No newline at end of file
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/batchnorm_inference.h b/deps/cudnn-frontend/include/cudnn_frontend/node/batchnorm_inference.h
new file mode 100644
index 0000000..9fb433f
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/node/batchnorm_inference.h
@@ -0,0 +1,156 @@
+#pragma once
+
+#include "../graph_helpers.h"
+#include "../node_interface.h"
+
+namespace cudnn_frontend {
+
+namespace graph {
+class BatchnormInferenceNode : public NodeCRTP<BatchnormInferenceNode> {
+ public:
+ Batchnorm_inference_attributes attributes;
+
+ BatchnormInferenceNode(Batchnorm_inference_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP(context), attributes(std::move(attributes_)) {}
+
+ Type
+ getType() override final {
+ return Type::BATCHNORM_INFERENCE;
+ }
+
+ error_t
+ infer_properties_node() override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for batchnorm inference node " << attributes.name);
+
+ attributes.fill_from_context(context);
+
+ auto X = attributes.inputs[Batchnorm_inference_attributes::input_names::X];
+ auto Y = attributes.outputs[Batchnorm_inference_attributes::output_names::Y];
+ // Only infer dims and strides if user did not set them
+ if (Y->get_dim().empty()) {
+ Y->set_dim(X->get_dim());
+ }
+ if (Y->get_stride().empty()) {
+ Y->set_stride(X->get_stride());
+ }
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ create_cudnn_operations(
+ std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations,
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations,
+ managed_backend_descriptor_t& raw_operations,
+ std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final {
+ CUDNN_FRONTEND_UNUSED(raw_operations);
+ CUDNN_FE_LOG_LABEL("INFO: Building BatchnormInferenceNode operations " << attributes.name << " ");
+
+ // Create operation by directly calling cuDNN backend API
+ Operation_v8 batchnorm_operation;
+
+ _CUDNN_CHECK_CUDNN_ERROR(
+ batchnorm_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_NORM_FORWARD_DESCRIPTOR));
+
+ // Set norm mode to BATCH_NORM
+ cudnnBackendNormMode_t cudnn_norm_mode;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormMode_t::BATCH_NORM, cudnn_norm_mode));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_MODE,
+ CUDNN_TYPE_NORM_MODE,
+ 1,
+ &cudnn_norm_mode));
+
+ // Set forward phase to INFERENCE
+ cudnnBackendNormFwdPhase_t cudnn_norm_fwd_phase;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormFwdPhase_t::INFERENCE, cudnn_norm_fwd_phase));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_PHASE,
+ CUDNN_TYPE_NORM_FWD_PHASE,
+ 1,
+ &cudnn_norm_fwd_phase));
+
+ // Set input tensor X
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Batchnorm_inference_attributes::input_names::X);
+ auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_XDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &x_desc));
+
+ // Set mean and inv_variance (as inputs for inference)
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(MEAN, Batchnorm_inference_attributes::input_names::MEAN);
+ auto mean_desc = tensors.at(MEAN->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_MEAN_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &mean_desc));
+
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(INV_VARIANCE,
+ Batchnorm_inference_attributes::input_names::INV_VARIANCE);
+ auto inv_var_desc = tensors.at(INV_VARIANCE->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_INV_VARIANCE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &inv_var_desc));
+
+ // Set scale and bias tensors
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SCALE, Batchnorm_inference_attributes::input_names::SCALE);
+ auto scale_desc = tensors.at(SCALE->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_SCALE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &scale_desc));
+
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(BIAS, Batchnorm_inference_attributes::input_names::BIAS);
+ auto bias_desc = tensors.at(BIAS->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_BIAS_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &bias_desc));
+
+ // Set output tensor Y
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(Y, Batchnorm_inference_attributes::output_names::Y);
+ auto y_desc = tensors.at(Y->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_YDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &y_desc));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(batchnorm_operation.get_raw_desc()));
+
+ operations.push_back(std::make_shared<Operation_v8>(std::move(batchnorm_operation)));
+
+ auto const& non_virtual_uids = attributes.get_non_virtual_uids();
+ uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end());
+ return {error_code_t::OK, ""};
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ j.update(R"( {"tag": "BATCHNORM_INFERENCE"})"_json);
+ }
+#endif
+};
+
+} // namespace graph
+
+} // namespace cudnn_frontend \ No newline at end of file
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/block_scale_dequantize.h b/deps/cudnn-frontend/include/cudnn_frontend/node/block_scale_dequantize.h
new file mode 100644
index 0000000..bbaa2bf
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/node/block_scale_dequantize.h
@@ -0,0 +1,169 @@
+#pragma once
+
+#include "../../cudnn_frontend_Logging.h"
+#include "../../cudnn_frontend_shim.h"
+
+#include "../graph_helpers.h"
+#include "../node_interface.h"
+
+namespace cudnn_frontend {
+
+namespace graph {
+
+class BlockScaleDequantizeNode : public NodeCRTP<BlockScaleDequantizeNode> {
+ public:
+ Block_scale_dequantize_attributes attributes;
+
+ BlockScaleDequantizeNode(Block_scale_dequantize_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP(context), attributes(std::move(attributes_)) {}
+
+ Type
+ getType() override final {
+ return Type::BLOCK_SCALE_DEQUANTIZE;
+ }
+
+ error_t
+ pre_validate_node() const override final {
+ getLogger() << "[cudnn_frontend] INFO: "
+ << "Validating BlockScaleDequantizeNode " << attributes.name << std::endl;
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ attributes.block_size.empty(), error_code_t::ATTRIBUTE_NOT_SET, "Block size not set\n");
+
+ auto Y = attributes.outputs.at(Block_scale_dequantize_attributes::output_names::Y);
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(!(Y->get_is_virtual()),
+ error_code_t::INVALID_VALUE,
+ "Output tensor of dequantize node should be virtual\n");
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ infer_properties_node() override final {
+ getLogger() << "[cudnn_frontend] INFO: Inferencing properties for BlockScaleDequantizeNode " << attributes.name
+ << std::endl;
+
+ attributes.fill_from_context(context);
+
+ auto X = attributes.inputs[Block_scale_dequantize_attributes::input_names::X];
+ auto scale = attributes.inputs[Block_scale_dequantize_attributes::input_names::scale];
+ auto Y = attributes.outputs[Block_scale_dequantize_attributes::output_names::Y];
+
+ // Only infer dims and strides if user did not set them
+ if (Y->get_dim().empty()) {
+ Y->set_dim(X->get_dim());
+ }
+
+ if (Y->get_stride().empty()) {
+ Y->set_stride(X->get_stride());
+ }
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ create_cudnn_operations(
+ std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations,
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations,
+ managed_backend_descriptor_t& raw_operations,
+ std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final {
+ getLogger() << "[cudnn_frontend] INFO: "
+ << "Building BlockScaleDequantizeNode operations " << attributes.name << std::endl;
+ auto cudnn_ver_error =
+ error_t{error_code_t::GRAPH_NOT_SUPPORTED, "Block scale dequantize requires cuDNN v9.7.0"};
+
+#if (CUDNN_VERSION >= 90700) // TODO: v9.99 is new feature branch; switch to release branch when ready
+ NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(90700, cudnn_ver_error);
+ CUDNN_FRONTEND_UNUSED(operations);
+ auto block_scale_dequantize_operation = make_shared_backend_pointer(
+ (cudnnBackendDescriptorType_t)CUDNN_BACKEND_OPERATION_BLOCK_SCALE_DEQUANTIZE_DESCRIPTOR);
+
+ auto X = attributes.inputs.find(Block_scale_dequantize_attributes::input_names::X)->second;
+ auto backend_x = tensors[X->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(block_scale_dequantize_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_BLOCK_SCALE_DEQUANTIZE_XDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_x));
+
+ auto scale = attributes.inputs.find(Block_scale_dequantize_attributes::input_names::scale)->second;
+ auto backend_scale = tensors[scale->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(block_scale_dequantize_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_BLOCK_SCALE_DEQUANTIZE_SCALE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_scale));
+
+ auto Y = attributes.outputs.find(Block_scale_dequantize_attributes::output_names::Y)->second;
+ auto backend_y = tensors[Y->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(block_scale_dequantize_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_BLOCK_SCALE_DEQUANTIZE_YDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_y));
+
+ cudnnDataType_t cudnn_data_type;
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.compute_data_type, cudnn_data_type));
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(block_scale_dequantize_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_BLOCK_SCALE_DEQUANTIZE_MATH_PREC,
+ CUDNN_TYPE_DATA_TYPE,
+ 1,
+ &cudnn_data_type));
+
+ const int32_t* block_size = attributes.block_size.data();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(block_scale_dequantize_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_BLOCK_SCALE_DEQUANTIZE_BLOCK_SIZE,
+ CUDNN_TYPE_INT32,
+ attributes.block_size.size(),
+ block_size));
+
+#if (CUDNN_VERSION >= 91400)
+ if (detail::get_backend_version() >= 91400) {
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(block_scale_dequantize_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_BLOCK_SCALE_DEQUANTIZE_NEG_SCALE,
+ CUDNN_TYPE_BOOLEAN,
+ 1,
+ &attributes.is_negative_scale));
+ }
+#endif
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(block_scale_dequantize_operation->get_backend_descriptor()));
+
+ raw_operations.push_back(block_scale_dequantize_operation);
+
+ auto const& non_virtual_uids = attributes.get_non_virtual_uids();
+ uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end());
+ return {error_code_t::OK, ""};
+#else
+ CUDNN_FRONTEND_UNUSED(uids_involved_in_operations);
+ CUDNN_FRONTEND_UNUSED(operations);
+ CUDNN_FRONTEND_UNUSED(raw_operations);
+ CUDNN_FRONTEND_UNUSED(tensors);
+ return cudnn_ver_error;
+#endif
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ j.update(R"( {"tag": "BLOCK_SCALE_DEQUANTIZE"})"_json);
+ }
+#endif
+};
+
+inline void
+INode::block_scale_dequantize(std::shared_ptr<Tensor_attributes> x,
+ std::shared_ptr<Tensor_attributes> scale,
+ Block_scale_dequantize_attributes attributes,
+ std::shared_ptr<Tensor_attributes> y) {
+ attributes.inputs[Block_scale_dequantize_attributes::input_names::X] = x;
+ attributes.inputs[Block_scale_dequantize_attributes::input_names::scale] = scale;
+ attributes.outputs[Block_scale_dequantize_attributes::output_names::Y] = y;
+ sub_nodes.emplace_back(std::make_unique<BlockScaleDequantizeNode>(std::move(attributes), context));
+}
+
+} // namespace graph
+
+} // namespace cudnn_frontend \ No newline at end of file
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/block_scale_quantize.h b/deps/cudnn-frontend/include/cudnn_frontend/node/block_scale_quantize.h
new file mode 100644
index 0000000..494ea14
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/node/block_scale_quantize.h
@@ -0,0 +1,205 @@
+#pragma once
+
+#include "../../cudnn_frontend_Logging.h"
+#include "../../cudnn_frontend_shim.h"
+
+#include "../graph_helpers.h"
+#include "../node_interface.h"
+
+namespace cudnn_frontend {
+
+namespace graph {
+
+class BlockScaleQuantizeNode : public NodeCRTP<BlockScaleQuantizeNode> {
+ public:
+ Block_scale_quantize_attributes attributes;
+
+ BlockScaleQuantizeNode(Block_scale_quantize_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP(context), attributes(std::move(attributes_)) {}
+
+ Type
+ getType() override final {
+ return Type::BLOCK_SCALE_QUANTIZE;
+ }
+
+ error_t
+ pre_validate_node() const override final {
+ getLogger() << "[cudnn_frontend] INFO: " << "Validating BlockScaleQuantizeNode " << attributes.name
+ << std::endl;
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ !attributes.block_size.has_value(), error_code_t::ATTRIBUTE_NOT_SET, "Block size not set.");
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ infer_properties_node() override final {
+ getLogger() << "[cudnn_frontend] INFO: Inferencing properties for BlockScaleQuantizeNode " << attributes.name
+ << std::endl;
+
+ attributes.fill_from_context(context);
+
+ auto X = attributes.inputs[Block_scale_quantize_attributes::input_names::X];
+ auto Y = attributes.outputs[Block_scale_quantize_attributes::output_names::Y];
+ auto scale = attributes.outputs[Block_scale_quantize_attributes::output_names::scale];
+
+ // Block scale quantize requires the block scale axis to be packed
+ auto infer_strides_transposed = [&X](std::shared_ptr<Tensor_attributes>& T,
+ std::optional<int64_t> const& axis) {
+ auto const& dim = T->get_dim();
+ auto const& X_dim = X->get_dim();
+ auto const& X_stride = X->get_stride();
+
+ std::vector<int64_t> indices(X_stride.size());
+ std::iota(indices.begin(), indices.end(), 0);
+ // Sort indices based on stride values in descending order
+ std::sort(indices.begin(), indices.end(), [&X_dim, &X_stride](int64_t i, int64_t j) {
+ // Prioritize singleton dimensions
+ if (X_stride[i] == X_stride[j]) {
+ return (X_dim[i] == 1) && (X_dim[j] != 1);
+ }
+ return X_stride[i] < X_stride[j];
+ });
+ if (axis) {
+ // Rotate left until the axis is the packed dim
+ std::rotate(indices.begin(), std::find(indices.begin(), indices.end(), axis.value()), indices.end());
+ }
+ std::vector<int64_t> stride_order(X_stride.size());
+ for (size_t i = 0; i < indices.size(); ++i) {
+ stride_order[indices[i]] = i;
+ }
+ T->set_stride(detail::generate_stride(dim, stride_order));
+ };
+
+ // Only infer dims and strides if user did not set them
+ if (Y->get_dim().empty()) {
+ Y->set_dim(X->get_dim());
+ }
+ if (Y->get_stride().empty()) {
+ if (attributes.transpose) {
+ infer_strides_transposed(Y, attributes.axis);
+ } else {
+ Y->set_stride(X->get_stride());
+ }
+ }
+
+ // Only infer dims and strides if user did not set them
+ if (scale->get_dim().empty()) {
+ auto scale_dim = X->get_dim();
+ if (attributes.axis) {
+ scale_dim[attributes.axis.value()] /= attributes.block_size.value();
+ } else {
+ scale_dim.back() /= attributes.block_size.value();
+ }
+ scale->set_dim(scale_dim);
+ }
+ if (scale->get_stride().empty()) {
+ if (attributes.transpose) {
+ infer_strides_transposed(scale, attributes.axis);
+ } else {
+ auto const& scale_dim = scale->get_dim();
+ std::vector<int64_t> stride_order;
+ CHECK_CUDNN_FRONTEND_ERROR(
+ detail::generate_stride_order_preserving_format(X->get_stride(), scale_dim.size(), stride_order));
+ scale->set_stride(detail::generate_stride(scale_dim, stride_order));
+ }
+ }
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ create_cudnn_operations(
+ std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations,
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations,
+ managed_backend_descriptor_t& raw_operations,
+ std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final {
+ getLogger() << "[cudnn_frontend] INFO: " << "Building BlockScaleQuantizeNode operations " << attributes.name
+ << std::endl;
+ auto cudnn_ver_error = error_t{error_code_t::GRAPH_NOT_SUPPORTED, "Block scale quantize requires cuDNN v9.7.0"};
+
+#if (CUDNN_VERSION >= 90700) // TODO: v9.99 is new feature branch; switch to release branch when ready
+ NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(90700, cudnn_ver_error);
+ CUDNN_FRONTEND_UNUSED(operations);
+ auto block_scale_quantize_operation = make_shared_backend_pointer(
+ (cudnnBackendDescriptorType_t)CUDNN_BACKEND_OPERATION_BLOCK_SCALE_QUANTIZE_DESCRIPTOR);
+
+ auto X = attributes.inputs.find(Block_scale_quantize_attributes::input_names::X)->second;
+ auto backend_x = tensors[X->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(block_scale_quantize_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_BLOCK_SCALE_QUANTIZE_XDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_x));
+
+ auto Y = attributes.outputs.find(Block_scale_quantize_attributes::output_names::Y)->second;
+ auto backend_y = tensors[Y->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(block_scale_quantize_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_BLOCK_SCALE_QUANTIZE_YDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_y));
+
+ auto scale = attributes.outputs.find(Block_scale_quantize_attributes::output_names::scale)->second;
+ auto backend_scale = tensors[scale->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(block_scale_quantize_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_BLOCK_SCALE_QUANTIZE_SCALE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_scale));
+
+ cudnnDataType_t cudnn_data_type;
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.compute_data_type, cudnn_data_type));
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(block_scale_quantize_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_BLOCK_SCALE_QUANTIZE_MATH_PREC,
+ CUDNN_TYPE_DATA_TYPE,
+ 1,
+ &cudnn_data_type));
+
+ int32_t block_size = attributes.block_size.value();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(block_scale_quantize_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_BLOCK_SCALE_QUANTIZE_BLOCK_SIZE,
+ CUDNN_TYPE_INT32,
+ 1,
+ &block_size));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(block_scale_quantize_operation->get_backend_descriptor()));
+
+ raw_operations.push_back(block_scale_quantize_operation);
+
+ auto const& non_virtual_uids = attributes.get_non_virtual_uids();
+ uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end());
+ return {error_code_t::OK, ""};
+#else
+ CUDNN_FRONTEND_UNUSED(uids_involved_in_operations);
+ CUDNN_FRONTEND_UNUSED(operations);
+ CUDNN_FRONTEND_UNUSED(raw_operations);
+ CUDNN_FRONTEND_UNUSED(tensors);
+ return cudnn_ver_error;
+#endif
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ j.update(R"( {"tag": "BLOCK_SCALE_QUANTIZE"})"_json);
+ }
+#endif
+};
+
+inline void
+INode::block_scale_quantize(std::shared_ptr<Tensor_attributes> x,
+ Block_scale_quantize_attributes attributes,
+ std::shared_ptr<Tensor_attributes> y,
+ std::shared_ptr<Tensor_attributes> scale) {
+ attributes.inputs[Block_scale_quantize_attributes::input_names::X] = x;
+ attributes.outputs[Block_scale_quantize_attributes::output_names::Y] = y;
+ attributes.outputs[Block_scale_quantize_attributes::output_names::scale] = scale;
+ sub_nodes.emplace_back(std::make_unique<BlockScaleQuantizeNode>(std::move(attributes), context));
+}
+
+} // namespace graph
+
+} // namespace cudnn_frontend
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/bn_finalize.h b/deps/cudnn-frontend/include/cudnn_frontend/node/bn_finalize.h
new file mode 100644
index 0000000..99d3ec0
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/node/bn_finalize.h
@@ -0,0 +1,263 @@
+#pragma once
+
+#include "../graph_helpers.h"
+#include "../node_interface.h"
+
+namespace cudnn_frontend {
+
+namespace graph {
+
+class BatchNormFinalizeNode : public NodeCRTP<BatchNormFinalizeNode> {
+ public:
+ BN_finalize_attributes attributes;
+
+ BatchNormFinalizeNode(BN_finalize_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP(context), attributes(std::move(attributes_)) {}
+
+ Type
+ getType() override final {
+ return Type::BN_FINALIZE;
+ }
+
+ error_t
+ infer_properties_node() override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO:Inferencing properties for batchnorm finalize node " << attributes.name);
+
+ attributes.fill_from_context(context);
+
+ auto SUM = attributes.inputs[BN_finalize_attributes::input_names::SUM];
+ auto const sum_tensor_dim = SUM->get_dim();
+
+ // Set channel length tensors
+ auto infer_per_channel_tensors = [&sum_tensor_dim](std::shared_ptr<Tensor_attributes>& T) {
+ auto tensor_dim = T->get_dim();
+ // Only infer dims and strides if user did not set them
+ if (tensor_dim.empty()) {
+ tensor_dim = sum_tensor_dim;
+ T->set_dim(tensor_dim);
+ }
+ if (T->get_stride().empty()) {
+ auto const& T_dim = T->get_dim();
+ // Default to NHWC
+ auto const& stride_order = detail::generate_NHWC_stride_order(T_dim.size());
+ T->set_stride(detail::generate_stride(T_dim, stride_order));
+ }
+ };
+ infer_per_channel_tensors(attributes.outputs[BN_finalize_attributes::output_names::EQ_BIAS]);
+ infer_per_channel_tensors(attributes.outputs[BN_finalize_attributes::output_names::EQ_SCALE]);
+ infer_per_channel_tensors(attributes.outputs[BN_finalize_attributes::output_names::MEAN]);
+ infer_per_channel_tensors(attributes.outputs[BN_finalize_attributes::output_names::INV_VARIANCE]);
+ infer_per_channel_tensors(attributes.outputs[BN_finalize_attributes::output_names::NEXT_RUNNING_MEAN]);
+ infer_per_channel_tensors(attributes.outputs[BN_finalize_attributes::output_names::NEXT_RUNNING_VAR]);
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ create_cudnn_operations(
+ std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations,
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations,
+ managed_backend_descriptor_t& raw_operations,
+ std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final {
+ CUDNN_FRONTEND_UNUSED(raw_operations);
+ CUDNN_FE_LOG_LABEL("INFO: " << "Building BatchNormFinalizeNode operations " << attributes.name << " ");
+
+ // Create operation by directly calling cuDNN backend API
+ Operation_v8 bn_finalize_operation;
+
+ _CUDNN_CHECK_CUDNN_ERROR(bn_finalize_operation.initialize_managed_backend_pointer(
+ CUDNN_BACKEND_OPERATION_BN_FINALIZE_STATISTICS_DESCRIPTOR));
+
+ // Set BN finalize mode
+ cudnnBnFinalizeStatsMode_t bn_finalize_mode = CUDNN_BN_FINALIZE_STATISTICS_TRAINING;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_BN_FINALIZE_STATS_MODE,
+ CUDNN_TYPE_BN_FINALIZE_STATS_MODE,
+ 1,
+ &bn_finalize_mode));
+
+ // Set compute type (math precision)
+ cudnnDataType_t compute_type = CUDNN_DATA_FLOAT;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_BN_FINALIZE_MATH_PREC,
+ CUDNN_TYPE_DATA_TYPE,
+ 1,
+ &compute_type));
+
+ // Set SUM input tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SUM, BN_finalize_attributes::input_names::SUM);
+ auto sum_desc = tensors.at(SUM->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_BN_FINALIZE_Y_SUM_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &sum_desc));
+
+ // Set SQ_SUM input tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SQ_SUM, BN_finalize_attributes::input_names::SQ_SUM);
+ auto sq_sum_desc = tensors.at(SQ_SUM->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_BN_FINALIZE_Y_SQ_SUM_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &sq_sum_desc));
+
+ // Set SCALE input tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SCALE, BN_finalize_attributes::input_names::SCALE);
+ auto scale_desc = tensors.at(SCALE->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_BN_FINALIZE_SCALE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &scale_desc));
+
+ // Set BIAS input tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(BIAS, BN_finalize_attributes::input_names::BIAS);
+ auto bias_desc = tensors.at(BIAS->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_BN_FINALIZE_BIAS_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &bias_desc));
+
+ // Set EQ_SCALE output tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(EQ_SCALE, BN_finalize_attributes::output_names::EQ_SCALE);
+ auto eq_scale_desc = tensors.at(EQ_SCALE->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_BN_FINALIZE_EQ_SCALE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &eq_scale_desc));
+
+ // Set EQ_BIAS output tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(EQ_BIAS, BN_finalize_attributes::output_names::EQ_BIAS);
+ auto eq_bias_desc = tensors.at(EQ_BIAS->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_BN_FINALIZE_EQ_BIAS_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &eq_bias_desc));
+
+ // Set PREV_RUNNING_MEAN input tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(PREV_RUNNING_MEAN,
+ BN_finalize_attributes::input_names::PREV_RUNNING_MEAN);
+ auto prev_running_mean_desc = tensors.at(PREV_RUNNING_MEAN->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_BN_FINALIZE_PREV_RUNNING_MEAN_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &prev_running_mean_desc));
+
+ // Set PREV_RUNNING_VAR input tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(PREV_RUNNING_VAR,
+ BN_finalize_attributes::input_names::PREV_RUNNING_VAR);
+ auto prev_running_var_desc = tensors.at(PREV_RUNNING_VAR->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_BN_FINALIZE_PREV_RUNNING_VAR_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &prev_running_var_desc));
+
+ // Set NEXT_RUNNING_MEAN output tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(NEXT_RUNNING_MEAN,
+ BN_finalize_attributes::output_names::NEXT_RUNNING_MEAN);
+ auto next_running_mean_desc = tensors.at(NEXT_RUNNING_MEAN->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_BN_FINALIZE_UPDATED_RUNNING_MEAN_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &next_running_mean_desc));
+
+ // Set NEXT_RUNNING_VAR output tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(NEXT_RUNNING_VAR,
+ BN_finalize_attributes::output_names::NEXT_RUNNING_VAR);
+ auto next_running_var_desc = tensors.at(NEXT_RUNNING_VAR->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_BN_FINALIZE_UPDATED_RUNNING_VAR_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &next_running_var_desc));
+
+ // Set MEAN output tensor (saved mean)
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(MEAN, BN_finalize_attributes::output_names::MEAN);
+ auto mean_desc = tensors.at(MEAN->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_BN_FINALIZE_SAVED_MEAN_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &mean_desc));
+
+ // Set INV_VARIANCE output tensor (saved inv std)
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(INV_VARIANCE, BN_finalize_attributes::output_names::INV_VARIANCE);
+ auto inv_var_desc = tensors.at(INV_VARIANCE->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_BN_FINALIZE_SAVED_INV_STD_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &inv_var_desc));
+
+ // Set EPSILON tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(EPSILON, BN_finalize_attributes::input_names::EPSILON);
+ auto epsilon_desc = tensors.at(EPSILON->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_BN_FINALIZE_EPSILON_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &epsilon_desc));
+
+ // Set MOMENTUM tensor (exp average factor)
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(MOMENTUM, BN_finalize_attributes::input_names::MOMENTUM);
+ auto momentum_desc = tensors.at(MOMENTUM->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_BN_FINALIZE_EXP_AVERATE_FACTOR_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &momentum_desc));
+
+ // Set ACCUM_COUNT tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(ACCUM_COUNT, BN_finalize_attributes::input_names::ACCUM_COUNT);
+ auto accum_count_desc = tensors.at(ACCUM_COUNT->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_BN_FINALIZE_ACCUM_COUNT_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &accum_count_desc));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(bn_finalize_operation.get_raw_desc()));
+
+ operations.push_back(std::make_shared<Operation_v8>(std::move(bn_finalize_operation)));
+
+ auto const& non_virtual_uids = attributes.get_non_virtual_uids();
+ uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end());
+ return {error_code_t::OK, ""};
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ j.update(R"( {"tag": "BN_FINALIZE"})"_json);
+ }
+#endif
+};
+
+} // namespace graph
+
+} // namespace cudnn_frontend \ No newline at end of file
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/concatenate.h b/deps/cudnn-frontend/include/cudnn_frontend/node/concatenate.h
new file mode 100644
index 0000000..5f62589
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/node/concatenate.h
@@ -0,0 +1,163 @@
+#pragma once
+
+#include "../../cudnn_frontend_Logging.h"
+#include "../../cudnn_frontend_shim.h"
+
+#include "../graph_helpers.h"
+#include "../node_interface.h"
+#include <string>
+
+namespace cudnn_frontend {
+
+namespace graph {
+
+class ConcatenateNode : public NodeCRTP<ConcatenateNode> {
+ public:
+ Concatenate_attributes attributes;
+
+ ConcatenateNode(Concatenate_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP(context), attributes(std::move(attributes_)) {}
+
+ Type
+ getType() override final {
+ return Type::CONCATENATE;
+ }
+
+ error_t
+ pre_validate_node() const override final {
+ getLogger() << "[cudnn_frontend] INFO: " << "Validating ConcatenateNode " << attributes.name << std::endl;
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(!attributes.axis.has_value(), error_code_t::ATTRIBUTE_NOT_SET, "Axis not set\n");
+
+ auto X = attributes.inputs;
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ (X.size() == 0), error_code_t::INVALID_VALUE, "Input size of the concatenate node cannot be zero\n");
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ infer_properties_node() override final {
+ getLogger() << "[cudnn_frontend] INFO: Inferring properties for ConcatenateNode " << attributes.name
+ << std::endl;
+
+ attributes.fill_from_context(context);
+
+ auto Y = attributes.outputs[Concatenate_attributes::output_names::Y];
+
+ // Infer dims and strides only if user did not set them
+ int64_t dim_sum = 0;
+ for (const auto& input : attributes.inputs) {
+ dim_sum += input->get_dim()[attributes.axis.value()];
+ }
+
+ auto X = attributes.inputs[0];
+ auto dims = X->get_dim();
+ dims[attributes.axis.value()] = dim_sum;
+
+ if (Y->get_dim().empty()) {
+ Y->set_dim(dims);
+ Y->set_dim(dims);
+ }
+
+ if (Y->get_stride().empty()) {
+ std::vector<int64_t> stride_order;
+ CHECK_CUDNN_FRONTEND_ERROR(
+ detail::generate_stride_order_preserving_format(X->get_stride(), dims.size(), stride_order));
+ Y->set_stride(detail::generate_stride(dims, stride_order));
+ }
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ create_cudnn_operations(
+ std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations,
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations,
+ managed_backend_descriptor_t& raw_operations,
+ std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final {
+ getLogger() << "[cudnn_frontend] INFO: " << "Building ConcatenateNode operations " << attributes.name
+ << std::endl;
+ auto cudnn_ver_error = error_t{error_code_t::GRAPH_NOT_SUPPORTED, "Concatenate requires cuDNN v9.7.0"};
+
+#if (CUDNN_VERSION >= 90700)
+ NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(90700, cudnn_ver_error);
+ CUDNN_FRONTEND_UNUSED(operations);
+ auto concatenate_operation = make_shared_backend_pointer(CUDNN_BACKEND_OPERATION_CONCAT_DESCRIPTOR);
+
+ std::vector<void*> backend_x(attributes.inputs.size());
+ size_t index = 0;
+ for (const auto& input : attributes.inputs) {
+ backend_x[index] = tensors[input->get_uid()]->get_desc()->get_backend_descriptor();
+ index++;
+ }
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(concatenate_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_CONCAT_INPUT_DESCS,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ attributes.inputs.size(),
+ backend_x.data()));
+
+ auto Y = attributes.outputs.find(Concatenate_attributes::output_names::Y)->second;
+ auto backend_y = tensors[Y->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(concatenate_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_CONCAT_OUTPUT_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_y));
+
+ int64_t axis = attributes.axis.value();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(concatenate_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_CONCAT_AXIS,
+ CUDNN_TYPE_INT64,
+ 1,
+ &axis));
+
+ if (attributes.in_place_index.has_value()) {
+ int64_t in_place_index = attributes.in_place_index.value();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(concatenate_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_CONCAT_INPLACE_INDEX,
+ CUDNN_TYPE_INT64,
+ 1,
+ &in_place_index));
+ }
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(concatenate_operation->get_backend_descriptor()));
+
+ raw_operations.push_back(concatenate_operation);
+
+ auto const& non_virtual_uids = attributes.get_non_virtual_uids();
+ uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end());
+ return {error_code_t::OK, ""};
+#else
+ CUDNN_FRONTEND_UNUSED(uids_involved_in_operations);
+ CUDNN_FRONTEND_UNUSED(operations);
+ CUDNN_FRONTEND_UNUSED(raw_operations);
+ CUDNN_FRONTEND_UNUSED(tensors);
+ return cudnn_ver_error;
+#endif
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ j.update(R"( {"tag": "CONCATENATE"})"_json);
+ }
+#endif
+};
+
+inline void
+INode::concatenate(std::vector<std::shared_ptr<Tensor_attributes>> x,
+ Concatenate_attributes attributes,
+ std::shared_ptr<Tensor_attributes> y) {
+ for (auto& element : x) {
+ attributes.inputs.push_back(element);
+ }
+ attributes.outputs[Concatenate_attributes::output_names::Y] = y;
+ sub_nodes.emplace_back(std::make_unique<ConcatenateNode>(std::move(attributes), context));
+}
+
+} // namespace graph
+
+} // namespace cudnn_frontend
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/conv_dgrad.h b/deps/cudnn-frontend/include/cudnn_frontend/node/conv_dgrad.h
new file mode 100644
index 0000000..a2e2e4e
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/node/conv_dgrad.h
@@ -0,0 +1,205 @@
+#pragma once
+
+#include "../graph_helpers.h"
+#include "../node_interface.h"
+
+namespace cudnn_frontend::graph {
+
+class DgradNode : public NodeCRTP<DgradNode> {
+ public:
+ Conv_dgrad_attributes attributes;
+
+ DgradNode(Conv_dgrad_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP(context), attributes(std::move(attributes_)) {}
+
+ Type
+ getType() override final {
+ return Type::DGRAD;
+ }
+
+ error_t
+ pre_validate_node() const override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating Node Type::DGRAD " << attributes.name);
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ attributes.get_pre_padding().empty(), error_code_t::ATTRIBUTE_NOT_SET, "Pre padding not set.");
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ attributes.get_post_padding().empty(), error_code_t::ATTRIBUTE_NOT_SET, "Post padding not set.");
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ attributes.get_stride().empty(), error_code_t::ATTRIBUTE_NOT_SET, "Conv strides not set.");
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ attributes.get_dilation().empty(), error_code_t::ATTRIBUTE_NOT_SET, "Conv dilation not set.");
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ infer_properties_node() override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for dgrad node " << attributes.name);
+
+ attributes.fill_from_context(context);
+
+ // TODO: Only inferrencing from (X, DY) -> DW works today.
+ auto DX = attributes.outputs.find(Conv_dgrad_attributes::output_names::DX)->second;
+ auto W = attributes.inputs.find(Conv_dgrad_attributes::input_names::W)->second;
+ auto DY = attributes.inputs.find(Conv_dgrad_attributes::input_names::DY)->second;
+
+ auto const w_tensor_dim = W->get_dim();
+ auto const dy_tensor_dim = DY->get_dim();
+ auto dx_tensor_dim = DX->get_dim();
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(DX->get_dim().empty(),
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "For dgrad node, output dimension inferencing is not possible.");
+
+ // No dim inferencing as inverse mapping from DY, W to DX is not unique.
+ // Only infer strides if user did not set them
+ if (DX->get_stride().empty()) {
+ auto const& DX_dim = DX->get_dim();
+ // Default to NHWC
+ auto const& stride_order = detail::generate_NHWC_stride_order(DX_dim.size());
+ DX->set_stride(detail::generate_stride(DX_dim, stride_order));
+ }
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ create_cudnn_operations(
+ std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations,
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations,
+ managed_backend_descriptor_t& raw_operations,
+ std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final {
+ CUDNN_FRONTEND_UNUSED(raw_operations);
+ CUDNN_FE_LOG_LABEL("INFO: Building DgradNode operations " << attributes.name << " ");
+
+ // Create dgrad descriptor by directly calling cuDNN backend API
+ ConvDesc_v8 dgrad_descriptor;
+ int64_t const spatial_dim_count = attributes.get_pre_padding().size();
+
+ _CUDNN_CHECK_CUDNN_ERROR(
+ dgrad_descriptor.initialize_managed_backend_pointer(CUDNN_BACKEND_CONVOLUTION_DESCRIPTOR));
+
+ cudnnDataType_t cudnn_data_type;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.compute_data_type, cudnn_data_type));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dgrad_descriptor.get_raw_desc(),
+ CUDNN_ATTR_CONVOLUTION_COMP_TYPE,
+ CUDNN_TYPE_DATA_TYPE,
+ 1,
+ &cudnn_data_type));
+
+ cudnnConvolutionMode_t mode = detail::convert_to_cudnn_type(attributes.math_mode);
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(
+ dgrad_descriptor.get_raw_desc(), CUDNN_ATTR_CONVOLUTION_CONV_MODE, CUDNN_TYPE_CONVOLUTION_MODE, 1, &mode));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dgrad_descriptor.get_raw_desc(),
+ CUDNN_ATTR_CONVOLUTION_SPATIAL_DIMS,
+ CUDNN_TYPE_INT64,
+ 1,
+ &spatial_dim_count));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dgrad_descriptor.get_raw_desc(),
+ CUDNN_ATTR_CONVOLUTION_PRE_PADDINGS,
+ CUDNN_TYPE_INT64,
+ spatial_dim_count,
+ attributes.get_pre_padding().data()));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dgrad_descriptor.get_raw_desc(),
+ CUDNN_ATTR_CONVOLUTION_POST_PADDINGS,
+ CUDNN_TYPE_INT64,
+ spatial_dim_count,
+ attributes.get_post_padding().data()));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dgrad_descriptor.get_raw_desc(),
+ CUDNN_ATTR_CONVOLUTION_DILATIONS,
+ CUDNN_TYPE_INT64,
+ spatial_dim_count,
+ attributes.get_dilation().data()));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dgrad_descriptor.get_raw_desc(),
+ CUDNN_ATTR_CONVOLUTION_FILTER_STRIDES,
+ CUDNN_TYPE_INT64,
+ spatial_dim_count,
+ attributes.get_stride().data()));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(dgrad_descriptor.get_raw_desc()));
+ CUDNN_FE_LOG_LABEL_ENDL(dgrad_descriptor);
+
+ // Create operation by directly calling cuDNN backend API
+ Operation_v8 dgrad_operation;
+
+ _CUDNN_CHECK_CUDNN_ERROR(dgrad_operation.initialize_managed_backend_pointer(
+ CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_DATA_DESCRIPTOR));
+
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DX, Conv_dgrad_attributes::output_names::DX);
+ auto dx_desc = tensors.at(DX->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dgrad_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_DATA_DX,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &dx_desc));
+
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(W, Conv_dgrad_attributes::input_names::W);
+ auto w_desc = tensors.at(W->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dgrad_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_DATA_W,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &w_desc));
+
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(DY, Conv_dgrad_attributes::input_names::DY);
+ auto dy_desc = tensors.at(DY->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dgrad_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_DATA_DY,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &dy_desc));
+
+ auto conv_desc_ptr = dgrad_descriptor.get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dgrad_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_DATA_CONV_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &conv_desc_ptr));
+
+ float alpha = 1.0f;
+ float beta = 0.0f;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dgrad_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_DATA_ALPHA,
+ CUDNN_TYPE_FLOAT,
+ 1,
+ &alpha));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dgrad_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_DATA_BETA,
+ CUDNN_TYPE_FLOAT,
+ 1,
+ &beta));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(dgrad_operation.get_raw_desc()));
+
+ operations.push_back(std::make_shared<Operation_v8>(std::move(dgrad_operation)));
+
+ auto const& non_virtual_uids = attributes.get_non_virtual_uids();
+ uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end());
+ return {error_code_t::OK, ""};
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ j.update(R"( {"tag": "CONV_DGRAD"})"_json);
+ }
+#endif
+};
+
+} // namespace cudnn_frontend::graph \ No newline at end of file
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/conv_fprop.h b/deps/cudnn-frontend/include/cudnn_frontend/node/conv_fprop.h
new file mode 100644
index 0000000..34acf55
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/node/conv_fprop.h
@@ -0,0 +1,254 @@
+#pragma once
+
+#include "../graph_helpers.h"
+#include "../node_interface.h"
+
+namespace cudnn_frontend::graph {
+class ConvolutionNode : public NodeCRTP<ConvolutionNode> {
+ public:
+ Conv_fprop_attributes attributes;
+
+ ConvolutionNode(Conv_fprop_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP(context), attributes(std::move(attributes_)) {}
+
+ Type
+ getType() override final {
+ return Type::CONVOLUTION;
+ }
+
+ error_t
+ pre_validate_node() const override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating Node Type::CONVOLUTION " << attributes.name);
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ attributes.get_pre_padding().empty(), error_code_t::ATTRIBUTE_NOT_SET, "Pre padding not set.");
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ attributes.get_post_padding().empty(), error_code_t::ATTRIBUTE_NOT_SET, "Post padding not set.");
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ attributes.get_stride().empty(), error_code_t::ATTRIBUTE_NOT_SET, "Conv strides not set.");
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ attributes.get_dilation().empty(), error_code_t::ATTRIBUTE_NOT_SET, "Conv dilation not set.");
+
+ // Implicit GEMM kernels compute B_offset as int32. When the product of all filter
+ // dimensions (K * C * spatial...) exceeds INT32_MAX the offset overflows, causing IMA or
+ // silent NaN. Conservative upper bound — exact check requires kernel tiling params not
+ // available at graph-validate time. Applies to 2D (4D filter) and 3D (5D filter) fprop.
+ auto W_it = attributes.inputs.find(Conv_fprop_attributes::input_names::W);
+ if (W_it != attributes.inputs.end() && W_it->second) {
+ auto const& w_dim = W_it->second->get_dim();
+ if (w_dim.size() == 4 || w_dim.size() == 5) {
+ int64_t filter_elements = 1;
+ for (auto d : w_dim) filter_elements *= d;
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ filter_elements > static_cast<int64_t>(std::numeric_limits<int32_t>::max()),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Conv filter total elements exceed INT32_MAX.");
+ }
+ }
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ infer_properties_node() override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for conv node " << attributes.name);
+
+ attributes.fill_from_context(context);
+
+ // TODO: Only inferrencing from (X, W) -> Y works today.
+ auto& X = attributes.inputs.find(Conv_fprop_attributes::input_names::X)->second;
+ auto& W = attributes.inputs.find(Conv_fprop_attributes::input_names::W)->second;
+ auto& Y = attributes.outputs.find(Conv_fprop_attributes::output_names::Y)->second;
+
+ auto const x_tensor_dim = X->get_dim();
+ auto const w_tensor_dim = W->get_dim();
+ auto y_tensor_dim = Y->get_dim();
+
+ // Only infer dims and strides if user did not set them
+ if (y_tensor_dim.empty()) {
+ y_tensor_dim.resize(x_tensor_dim.size());
+ auto const& pre_padding = attributes.get_pre_padding();
+ auto const& post_padding = attributes.get_post_padding();
+ auto const& stride = attributes.get_stride();
+ auto const& dilation = attributes.get_dilation();
+ // N
+ y_tensor_dim[0] = x_tensor_dim[0];
+ // PQ
+ for (size_t dim = 2; dim < x_tensor_dim.size(); ++dim) {
+ y_tensor_dim[dim] = 1 + (x_tensor_dim[dim] - dilation[dim - 2] * (w_tensor_dim[dim] - 1) - 1 +
+ pre_padding[dim - 2] + post_padding[dim - 2]) /
+ stride[dim - 2];
+ }
+ // K
+ y_tensor_dim[1] = w_tensor_dim[0];
+ Y->set_dim(y_tensor_dim);
+ }
+ if (Y->get_stride().empty()) {
+ auto const& Y_dim = Y->get_dim();
+ // Default to NHWC
+ auto const& stride_order = detail::generate_NHWC_stride_order(Y_dim.size());
+ Y->set_stride(detail::generate_stride(Y_dim, stride_order));
+ }
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ create_cudnn_operations(
+ std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations,
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations,
+ managed_backend_descriptor_t& raw_operations,
+ std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final {
+ CUDNN_FRONTEND_UNUSED(raw_operations);
+ CUDNN_FE_LOG_LABEL("INFO: Building ConvolutionNode operations " << attributes.name << " ");
+
+ // Create convolution descriptor by directly calling cuDNN backend API
+ ConvDesc_v8 convolution_descriptor;
+ int64_t const spatial_dim_count = attributes.get_pre_padding().size();
+
+ _CUDNN_CHECK_CUDNN_ERROR(
+ convolution_descriptor.initialize_managed_backend_pointer(CUDNN_BACKEND_CONVOLUTION_DESCRIPTOR));
+
+ // Set compute type
+ cudnnDataType_t cudnn_data_type;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.compute_data_type, cudnn_data_type));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_descriptor.get_raw_desc(),
+ CUDNN_ATTR_CONVOLUTION_COMP_TYPE,
+ CUDNN_TYPE_DATA_TYPE,
+ 1,
+ &cudnn_data_type));
+
+ // Set convolution mode
+ cudnnConvolutionMode_t mode = detail::convert_to_cudnn_type(attributes.math_mode);
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_descriptor.get_raw_desc(),
+ CUDNN_ATTR_CONVOLUTION_CONV_MODE,
+ CUDNN_TYPE_CONVOLUTION_MODE,
+ 1,
+ &mode));
+
+ // Set spatial dimensions
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_descriptor.get_raw_desc(),
+ CUDNN_ATTR_CONVOLUTION_SPATIAL_DIMS,
+ CUDNN_TYPE_INT64,
+ 1,
+ &spatial_dim_count));
+
+ // Set pre-padding
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_descriptor.get_raw_desc(),
+ CUDNN_ATTR_CONVOLUTION_PRE_PADDINGS,
+ CUDNN_TYPE_INT64,
+ spatial_dim_count,
+ attributes.get_pre_padding().data()));
+
+ // Set post-padding
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_descriptor.get_raw_desc(),
+ CUDNN_ATTR_CONVOLUTION_POST_PADDINGS,
+ CUDNN_TYPE_INT64,
+ spatial_dim_count,
+ attributes.get_post_padding().data()));
+
+ // Set dilation
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_descriptor.get_raw_desc(),
+ CUDNN_ATTR_CONVOLUTION_DILATIONS,
+ CUDNN_TYPE_INT64,
+ spatial_dim_count,
+ attributes.get_dilation().data()));
+
+ // Set strides
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_descriptor.get_raw_desc(),
+ CUDNN_ATTR_CONVOLUTION_FILTER_STRIDES,
+ CUDNN_TYPE_INT64,
+ spatial_dim_count,
+ attributes.get_stride().data()));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(convolution_descriptor.get_raw_desc()));
+ CUDNN_FE_LOG_LABEL_ENDL(convolution_descriptor);
+
+ // Create operation by directly calling cuDNN backend API
+ Operation_v8 convolution_operation;
+
+ _CUDNN_CHECK_CUDNN_ERROR(convolution_operation.initialize_managed_backend_pointer(
+ CUDNN_BACKEND_OPERATION_CONVOLUTION_FORWARD_DESCRIPTOR));
+
+ // Set input tensor X
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Conv_fprop_attributes::input_names::X);
+ auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_X,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &x_desc));
+
+ // Set weight tensor W
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(W, Conv_fprop_attributes::input_names::W);
+ auto w_desc = tensors.at(W->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_W,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &w_desc));
+
+ // Set output tensor Y
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(Y, Conv_fprop_attributes::output_names::Y);
+ auto y_desc = tensors.at(Y->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_Y,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &y_desc));
+
+ // Set convolution descriptor
+ auto conv_desc_ptr = convolution_descriptor.get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_CONV_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &conv_desc_ptr));
+
+ // Set alpha and beta
+ float alpha = 1.0f;
+ float beta = 0.0f;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_ALPHA,
+ CUDNN_TYPE_FLOAT,
+ 1,
+ &alpha));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_BETA,
+ CUDNN_TYPE_FLOAT,
+ 1,
+ &beta));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(convolution_operation.get_raw_desc()));
+
+ operations.push_back(std::make_shared<Operation_v8>(std::move(convolution_operation)));
+
+ auto const& non_virtual_uids = attributes.get_non_virtual_uids();
+ uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end());
+ return {error_code_t::OK, ""};
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ j.update(R"({"tag": "CONV_FPROP"})"_json);
+ }
+#endif
+};
+
+} // namespace cudnn_frontend::graph \ No newline at end of file
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/conv_wgrad.h b/deps/cudnn-frontend/include/cudnn_frontend/node/conv_wgrad.h
new file mode 100644
index 0000000..2f9b478
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/node/conv_wgrad.h
@@ -0,0 +1,201 @@
+#pragma once
+
+#include "../graph_helpers.h"
+#include "../node_interface.h"
+
+namespace cudnn_frontend::graph {
+
+class WgradNode : public NodeCRTP<WgradNode> {
+ public:
+ Conv_wgrad_attributes attributes;
+
+ WgradNode(Conv_wgrad_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP(context), attributes(std::move(attributes_)) {}
+
+ Type
+ getType() override final {
+ return Type::WGRAD;
+ }
+
+ error_t
+ pre_validate_node() const override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating Node Type::WGRAD " << attributes.name);
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ attributes.get_pre_padding().empty(), error_code_t::ATTRIBUTE_NOT_SET, "Pre padding not set.");
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ attributes.get_post_padding().empty(), error_code_t::ATTRIBUTE_NOT_SET, "Post padding not set.");
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ attributes.get_stride().empty(), error_code_t::ATTRIBUTE_NOT_SET, "Conv strides not set.");
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ attributes.get_dilation().empty(), error_code_t::ATTRIBUTE_NOT_SET, "Conv dilation not set.");
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ infer_properties_node() override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for conv node " << attributes.name);
+
+ attributes.fill_from_context(context);
+
+ // TODO: Only inferrencing from (X, DY) -> DW works today.
+ auto X = attributes.inputs[Conv_wgrad_attributes::input_names::X];
+ auto DW = attributes.outputs[Conv_wgrad_attributes::output_names::DW];
+ auto DY = attributes.inputs[Conv_wgrad_attributes::input_names::DY];
+
+ auto const x_tensor_dim = X->get_dim();
+ auto const dy_tensor_dim = DY->get_dim();
+ auto dw_tensor_dim = DW->get_dim();
+
+ // No dim inferencing as inverse mapping from DY, X to DX is not unique.
+ // Only infer strides if user did not set them
+ if (DW->get_stride().empty()) {
+ auto const& DW_dim = DW->get_dim();
+ // Default to NHWC
+ auto const& stride_order = detail::generate_NHWC_stride_order(DW_dim.size());
+ DW->set_stride(detail::generate_stride(DW_dim, stride_order));
+ }
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ create_cudnn_operations(
+ std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations,
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations,
+ managed_backend_descriptor_t& raw_operations,
+ std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final {
+ CUDNN_FRONTEND_UNUSED(raw_operations);
+ CUDNN_FE_LOG_LABEL("INFO: Building WgradNode operations " << attributes.name << " ");
+
+ // Create wgrad descriptor by directly calling cuDNN backend API
+ ConvDesc_v8 wgrad_descriptor;
+ int64_t const spatial_dim_count = attributes.get_pre_padding().size();
+
+ _CUDNN_CHECK_CUDNN_ERROR(
+ wgrad_descriptor.initialize_managed_backend_pointer(CUDNN_BACKEND_CONVOLUTION_DESCRIPTOR));
+
+ cudnnDataType_t cudnn_data_type;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.compute_data_type, cudnn_data_type));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(wgrad_descriptor.get_raw_desc(),
+ CUDNN_ATTR_CONVOLUTION_COMP_TYPE,
+ CUDNN_TYPE_DATA_TYPE,
+ 1,
+ &cudnn_data_type));
+
+ cudnnConvolutionMode_t mode = detail::convert_to_cudnn_type(attributes.math_mode);
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(
+ wgrad_descriptor.get_raw_desc(), CUDNN_ATTR_CONVOLUTION_CONV_MODE, CUDNN_TYPE_CONVOLUTION_MODE, 1, &mode));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(wgrad_descriptor.get_raw_desc(),
+ CUDNN_ATTR_CONVOLUTION_SPATIAL_DIMS,
+ CUDNN_TYPE_INT64,
+ 1,
+ &spatial_dim_count));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(wgrad_descriptor.get_raw_desc(),
+ CUDNN_ATTR_CONVOLUTION_PRE_PADDINGS,
+ CUDNN_TYPE_INT64,
+ spatial_dim_count,
+ attributes.get_pre_padding().data()));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(wgrad_descriptor.get_raw_desc(),
+ CUDNN_ATTR_CONVOLUTION_POST_PADDINGS,
+ CUDNN_TYPE_INT64,
+ spatial_dim_count,
+ attributes.get_post_padding().data()));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(wgrad_descriptor.get_raw_desc(),
+ CUDNN_ATTR_CONVOLUTION_DILATIONS,
+ CUDNN_TYPE_INT64,
+ spatial_dim_count,
+ attributes.get_dilation().data()));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(wgrad_descriptor.get_raw_desc(),
+ CUDNN_ATTR_CONVOLUTION_FILTER_STRIDES,
+ CUDNN_TYPE_INT64,
+ spatial_dim_count,
+ attributes.get_stride().data()));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(wgrad_descriptor.get_raw_desc()));
+ CUDNN_FE_LOG_LABEL_ENDL(wgrad_descriptor);
+
+ // Create operation by directly calling cuDNN backend API
+ Operation_v8 wgrad_operation;
+
+ _CUDNN_CHECK_CUDNN_ERROR(wgrad_operation.initialize_managed_backend_pointer(
+ CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_FILTER_DESCRIPTOR));
+
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Conv_wgrad_attributes::input_names::X);
+ auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(wgrad_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_FILTER_X,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &x_desc));
+
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(DY, Conv_wgrad_attributes::input_names::DY);
+ auto dy_desc = tensors.at(DY->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(wgrad_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_FILTER_DY,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &dy_desc));
+
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DW, Conv_wgrad_attributes::output_names::DW);
+ auto dw_desc = tensors.at(DW->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(wgrad_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_FILTER_DW,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &dw_desc));
+
+ auto conv_desc_ptr = wgrad_descriptor.get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(wgrad_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_FILTER_CONV_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &conv_desc_ptr));
+
+ float alpha = 1.0f;
+ float beta = 0.0f;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(wgrad_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_FILTER_ALPHA,
+ CUDNN_TYPE_FLOAT,
+ 1,
+ &alpha));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(wgrad_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_FILTER_BETA,
+ CUDNN_TYPE_FLOAT,
+ 1,
+ &beta));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(wgrad_operation.get_raw_desc()));
+
+ operations.push_back(std::make_shared<Operation_v8>(std::move(wgrad_operation)));
+
+ auto const& non_virtual_uids = attributes.get_non_virtual_uids();
+ uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end());
+ return {error_code_t::OK, ""};
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ j.update(R"( {"tag": "CONV_WGRAD"})"_json);
+ }
+#endif
+};
+
+} // namespace cudnn_frontend::graph \ No newline at end of file
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/dbn.h b/deps/cudnn-frontend/include/cudnn_frontend/node/dbn.h
new file mode 100644
index 0000000..9f2a23e
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/node/dbn.h
@@ -0,0 +1,206 @@
+#pragma once
+
+#include "../graph_helpers.h"
+#include "../node_interface.h"
+
+namespace cudnn_frontend {
+
+namespace graph {
+
+class DBNNode : public NodeCRTP<DBNNode> {
+ public:
+ Batchnorm_backward_attributes attributes;
+
+ DBNNode(Batchnorm_backward_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP(context), attributes(std::move(attributes_)) {}
+
+ Type
+ getType() override final {
+ return Type::DBN;
+ }
+
+ error_t
+ infer_properties_node() override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for DBN node " << attributes.name);
+
+ attributes.fill_from_context(context);
+
+ // TODO: Only inferencing from X works today.
+ auto X = attributes.inputs[Batchnorm_backward_attributes::input_names::X];
+ auto const x_tensor_dim = X->get_dim();
+
+ auto DX = attributes.outputs[Batchnorm_backward_attributes::output_names::DX];
+ auto dx_tensor_dim = DX->get_dim();
+ // Only infer dims and strides if user did not set them
+ if (dx_tensor_dim.empty()) {
+ dx_tensor_dim.resize(x_tensor_dim.size());
+ DX->set_dim(x_tensor_dim);
+ }
+ if (DX->get_stride().empty()) {
+ auto const& DX_dim = DX->get_dim();
+ // Default to NHWC
+ auto const& stride_order = detail::generate_NHWC_stride_order(DX_dim.size());
+ DX->set_stride(detail::generate_stride(DX_dim, stride_order));
+ }
+
+ // Set channel length tensors
+ auto infer_per_channel_tensors = [&x_tensor_dim](std::shared_ptr<Tensor_attributes>& T) {
+ auto tensor_dim = T->get_dim();
+ // Only infer dims and strides if user did not set them
+ if (tensor_dim.empty()) {
+ tensor_dim.resize(x_tensor_dim.size(), 1);
+ tensor_dim[1] = x_tensor_dim[1];
+ T->set_dim(tensor_dim);
+ }
+ if (T->get_stride().empty()) {
+ auto const& T_dim = T->get_dim();
+ // Default to NHWC
+ auto const& stride_order = detail::generate_NHWC_stride_order(T_dim.size());
+ T->set_stride(detail::generate_stride(T_dim, stride_order));
+ }
+ };
+ infer_per_channel_tensors(attributes.outputs[Batchnorm_backward_attributes::output_names::DSCALE]);
+ infer_per_channel_tensors(attributes.outputs[Batchnorm_backward_attributes::output_names::DBIAS]);
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ create_cudnn_operations(
+ std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations,
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations,
+ managed_backend_descriptor_t& raw_operations,
+ std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final {
+ CUDNN_FRONTEND_UNUSED(raw_operations);
+ CUDNN_FE_LOG_LABEL("INFO: " << "Building DBNNode operations " << attributes.name << " ");
+
+ // Create operation by directly calling cuDNN backend API
+ Operation_v8 dbn_operation;
+
+ _CUDNN_CHECK_CUDNN_ERROR(
+ dbn_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_NORM_BACKWARD_DESCRIPTOR));
+
+ // Set norm mode to BATCH_NORM
+ cudnnBackendNormMode_t cudnn_norm_mode;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormMode_t::BATCH_NORM, cudnn_norm_mode));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dbn_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_MODE,
+ CUDNN_TYPE_NORM_MODE,
+ 1,
+ &cudnn_norm_mode));
+
+ // Set input tensor X
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Batchnorm_backward_attributes::input_names::X);
+ auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dbn_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_XDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &x_desc));
+
+ // Set DY tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(DY, Batchnorm_backward_attributes::input_names::DY);
+ auto dy_desc = tensors.at(DY->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dbn_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_DYDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &dy_desc));
+
+ // Set scale tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SCALE, Batchnorm_backward_attributes::input_names::SCALE);
+ auto scale_desc = tensors.at(SCALE->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dbn_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_SCALE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &scale_desc));
+
+ // Set mean and inv_variance tensors
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(MEAN, Batchnorm_backward_attributes::input_names::MEAN);
+ auto mean_desc = tensors.at(MEAN->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dbn_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_MEAN_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &mean_desc));
+
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(INV_VARIANCE,
+ Batchnorm_backward_attributes::input_names::INV_VARIANCE);
+ auto inv_var_desc = tensors.at(INV_VARIANCE->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dbn_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_INV_VARIANCE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &inv_var_desc));
+
+ // Set DSCALE and DBIAS output tensors
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DSCALE, Batchnorm_backward_attributes::output_names::DSCALE);
+ auto dscale_desc = tensors.at(DSCALE->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dbn_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_DSCALE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &dscale_desc));
+
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DBIAS, Batchnorm_backward_attributes::output_names::DBIAS);
+ auto dbias_desc = tensors.at(DBIAS->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dbn_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_DBIAS_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &dbias_desc));
+
+ // Set DX output tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DX, Batchnorm_backward_attributes::output_names::DX);
+ auto dx_desc = tensors.at(DX->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dbn_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_DXDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &dx_desc));
+
+ // Set peer stat tensors if any
+ if (!attributes.peer_stats.empty()) {
+ std::vector<cudnnBackendDescriptor_t> peer_stat_descs;
+ for (auto const& peer_stat : attributes.peer_stats) {
+ peer_stat_descs.push_back(tensors.at(peer_stat->get_uid())->get_raw_desc());
+ }
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dbn_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_PEER_STAT_DESCS,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ peer_stat_descs.size(),
+ peer_stat_descs.data()));
+ }
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(dbn_operation.get_raw_desc()));
+
+ operations.push_back(std::make_shared<Operation_v8>(std::move(dbn_operation)));
+
+ auto const& non_virtual_uids = attributes.get_non_virtual_uids();
+ uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end());
+ return {error_code_t::OK, ""};
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ j.update(R"( {"tag": "DBN"})"_json);
+ }
+#endif
+};
+
+} // namespace graph
+
+} // namespace cudnn_frontend \ No newline at end of file
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/dbn_weight.h b/deps/cudnn-frontend/include/cudnn_frontend/node/dbn_weight.h
new file mode 100644
index 0000000..bfad0df
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/node/dbn_weight.h
@@ -0,0 +1,215 @@
+#pragma once
+
+#include "../graph_helpers.h"
+#include "../node_interface.h"
+
+namespace cudnn_frontend {
+
+namespace graph {
+
+class DBNWeightNode : public NodeCRTP<DBNWeightNode> {
+ public:
+ DBN_weight_attributes attributes;
+
+ DBNWeightNode(DBN_weight_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP(context), attributes(std::move(attributes_)) {}
+
+ Type
+ getType() override final {
+ return Type::DBN_WEIGHT;
+ }
+
+ error_t
+ infer_properties_node() override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for batchnorm finalize node " << attributes.name);
+
+ attributes.fill_from_context(context);
+
+ // TODO: Only inferencing from DY works today.
+ auto DY = attributes.inputs[DBN_weight_attributes::input_names::DY];
+ auto const dy_tensor_dim = DY->get_dim();
+
+ auto X = attributes.inputs[DBN_weight_attributes::input_names::X];
+ auto x_tensor_dim = X->get_dim();
+ // Only infer dims and strides if user did not set them
+ if (x_tensor_dim.empty()) {
+ x_tensor_dim.resize(dy_tensor_dim.size());
+ X->set_dim(dy_tensor_dim);
+ }
+ if (X->get_stride().empty()) {
+ auto const& X_dim = X->get_dim();
+ // Default to NHWC
+ auto const& stride_order = detail::generate_NHWC_stride_order(X_dim.size());
+ X->set_stride(detail::generate_stride(X_dim, stride_order));
+ }
+
+ // Set channel length tensors
+ auto infer_per_channel_tensors = [&dy_tensor_dim](std::shared_ptr<Tensor_attributes> const& T) {
+ auto tensor_dim = T->get_dim();
+ // Only infer dims and strides if user did not set them
+ if (T->get_dim().empty()) {
+ tensor_dim.resize(dy_tensor_dim.size(), 1);
+ tensor_dim[1] = dy_tensor_dim[1];
+ T->set_dim(tensor_dim);
+ }
+ if (T->get_stride().empty()) {
+ auto const& T_dim = T->get_dim();
+ // Default to NHWC
+ auto const& stride_order = detail::generate_NHWC_stride_order(T_dim.size());
+ T->set_stride(detail::generate_stride(T_dim, stride_order));
+ }
+ };
+ infer_per_channel_tensors(attributes.outputs[DBN_weight_attributes::output_names::DBIAS]);
+ infer_per_channel_tensors(attributes.outputs[DBN_weight_attributes::output_names::DSCALE]);
+ infer_per_channel_tensors(attributes.outputs[DBN_weight_attributes::output_names::EQ_BIAS]);
+ infer_per_channel_tensors(attributes.outputs[DBN_weight_attributes::output_names::EQ_SCALE_DY]);
+ infer_per_channel_tensors(attributes.outputs[DBN_weight_attributes::output_names::EQ_SCALE_X]);
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ create_cudnn_operations(
+ std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations,
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations,
+ managed_backend_descriptor_t& raw_operations,
+ std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final {
+ CUDNN_FRONTEND_UNUSED(raw_operations);
+ CUDNN_FE_LOG_LABEL("INFO: " << "Building DBNWeightNode operations " << attributes.name << " ");
+
+ // Create operation by directly calling cuDNN backend API
+ Operation_v8 bn_bwd_weight_operation;
+
+ _CUDNN_CHECK_CUDNN_ERROR(bn_bwd_weight_operation.initialize_managed_backend_pointer(
+ CUDNN_BACKEND_OPERATION_BN_BWD_WEIGHTS_DESCRIPTOR));
+
+ // Set compute type (math precision)
+ cudnnDataType_t compute_type = CUDNN_DATA_FLOAT;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_bwd_weight_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_MATH_PREC,
+ CUDNN_TYPE_DATA_TYPE,
+ 1,
+ &compute_type));
+
+ // Set input tensor X
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, DBN_weight_attributes::input_names::X);
+ auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_bwd_weight_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_X_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &x_desc));
+
+ // Set DY tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(DY, DBN_weight_attributes::input_names::DY);
+ auto dy_desc = tensors.at(DY->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_bwd_weight_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_DY_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &dy_desc));
+
+ // Set mean tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(MEAN, DBN_weight_attributes::input_names::MEAN);
+ auto mean_desc = tensors.at(MEAN->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_bwd_weight_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_MEAN_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &mean_desc));
+
+ // Set inv_variance tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(INV_VARIANCE, DBN_weight_attributes::input_names::INV_VARIANCE);
+ auto inv_var_desc = tensors.at(INV_VARIANCE->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_bwd_weight_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_INVSTD_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &inv_var_desc));
+
+ // Set scale tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SCALE, DBN_weight_attributes::input_names::SCALE);
+ auto scale_desc = tensors.at(SCALE->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_bwd_weight_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_BN_SCALE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &scale_desc));
+
+ // Set DSCALE output tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DSCALE, DBN_weight_attributes::output_names::DSCALE);
+ auto dscale_desc = tensors.at(DSCALE->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_bwd_weight_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_DBN_SCALE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &dscale_desc));
+
+ // Set DBIAS output tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DBIAS, DBN_weight_attributes::output_names::DBIAS);
+ auto dbias_desc = tensors.at(DBIAS->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_bwd_weight_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_DBN_BIAS_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &dbias_desc));
+
+ // Set EQ_SCALE_DY output tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(EQ_SCALE_DY, DBN_weight_attributes::output_names::EQ_SCALE_DY);
+ auto eq_scale_dy_desc = tensors.at(EQ_SCALE_DY->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_bwd_weight_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_EQ_DY_SCALE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &eq_scale_dy_desc));
+
+ // Set EQ_SCALE_X output tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(EQ_SCALE_X, DBN_weight_attributes::output_names::EQ_SCALE_X);
+ auto eq_scale_x_desc = tensors.at(EQ_SCALE_X->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_bwd_weight_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_EQ_X_SCALE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &eq_scale_x_desc));
+
+ // Set EQ_BIAS output tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(EQ_BIAS, DBN_weight_attributes::output_names::EQ_BIAS);
+ auto eq_bias_desc = tensors.at(EQ_BIAS->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_bwd_weight_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_EQ_BIAS,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &eq_bias_desc));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(bn_bwd_weight_operation.get_raw_desc()));
+
+ operations.push_back(std::make_shared<Operation_v8>(std::move(bn_bwd_weight_operation)));
+
+ auto const& non_virtual_uids = attributes.get_non_virtual_uids();
+ uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end());
+ return {error_code_t::OK, ""};
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ j.update(R"( {"tag": "DBN_WEIGHT"})"_json);
+ }
+#endif
+};
+
+} // namespace graph
+
+} // namespace cudnn_frontend \ No newline at end of file
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/diagonal_band_mask.h b/deps/cudnn-frontend/include/cudnn_frontend/node/diagonal_band_mask.h
new file mode 100644
index 0000000..243d1e3
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/node/diagonal_band_mask.h
@@ -0,0 +1,474 @@
+#pragma once
+
+#include "../graph_helpers.h"
+#include "../node_interface.h"
+
+#include "pointwise.h"
+#include "reduction.h"
+
+namespace cudnn_frontend::graph {
+
+// Base class for nodes that represent a diagonal band mask operation
+// (which is either a left-bound or a right-bound mask, but not both at the same time).
+template <typename DerivedT>
+class DiagonalBandMaskNodeBase : public NodeCRTP<DerivedT> {
+ protected:
+ using input_names = DiagonalBandMask_attributes::input_names;
+ using output_names = DiagonalBandMask_attributes::output_names;
+
+ public:
+ DiagonalBandMask_attributes attributes;
+
+ DiagonalBandMaskNodeBase(DiagonalBandMask_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP<DerivedT>(context), attributes(std::move(attributes_)) {}
+
+ error_t
+ pre_validate_node() const override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating DiagonalBandMaskNode " << attributes.name);
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(has_left_bound() && has_shift_right_bound(),
+ error_code_t::INVALID_VALUE,
+ "DiagonalBandMaskNode cannot have both left_bound and shift_right_bound");
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ (has_seq_len_q() || has_seq_len_kv()) && (has_cu_seq_len_q() || has_cu_seq_len_kv()),
+ error_code_t::INVALID_VALUE,
+ "SEQ_LEN_Q / SEQ_LEN_KV and CU_SEQ_LEN_Q / CU_SEQ_LEN_KV are mutually exclusive");
+
+ return pre_validate_node_extra();
+ }
+
+ virtual error_t
+ pre_validate_node_extra() const {
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ infer_properties_node() override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for DiagonalBandMask node " << attributes.name);
+
+ attributes.fill_from_context(this->context);
+
+ auto input_tensor = attributes.inputs.at(DiagonalBandMask_attributes::input_names::X);
+ auto output_tensor = attributes.outputs.at(DiagonalBandMask_attributes::output_names::Y);
+
+ // Only infer dims and strides if user did not set them
+ if (output_tensor->get_dim().empty()) {
+ output_tensor->set_dim(input_tensor->get_dim());
+ }
+
+ if (output_tensor->get_stride().empty()) {
+ output_tensor->set_stride(input_tensor->get_stride());
+ }
+
+ return {error_code_t::OK, ""};
+ }
+
+ protected:
+ bool
+ has_seq_len_q() const {
+ auto seq_len_Q_it = attributes.inputs.find(DiagonalBandMask_attributes::input_names::SEQ_LEN_Q);
+ return ((seq_len_Q_it) != attributes.inputs.end() && seq_len_Q_it->second != nullptr);
+ }
+
+ bool
+ has_seq_len_kv() const {
+ auto seq_len_KV_it = attributes.inputs.find(DiagonalBandMask_attributes::input_names::SEQ_LEN_KV);
+ return ((seq_len_KV_it) != attributes.inputs.end() && seq_len_KV_it->second != nullptr);
+ }
+
+ bool
+ has_cu_seq_len_q() const {
+ auto cu_seq_len_Q_it = attributes.inputs.find(DiagonalBandMask_attributes::input_names::CU_SEQ_LEN_Q);
+ return ((cu_seq_len_Q_it) != attributes.inputs.end() && cu_seq_len_Q_it->second != nullptr);
+ }
+
+ bool
+ has_cu_seq_len_kv() const {
+ auto cu_seq_len_KV_it = attributes.inputs.find(DiagonalBandMask_attributes::input_names::CU_SEQ_LEN_KV);
+ return ((cu_seq_len_KV_it) != attributes.inputs.end() && cu_seq_len_KV_it->second != nullptr);
+ }
+
+ bool
+ has_left_bound() const {
+ auto left_bound_it = attributes.inputs.find(DiagonalBandMask_attributes::input_names::LeftBound);
+ return ((left_bound_it) != attributes.inputs.end() && left_bound_it->second != nullptr);
+ }
+
+ bool
+ has_right_bound() const {
+ return !has_left_bound();
+ }
+
+ bool
+ has_shift_right_bound() const {
+ auto shift_right_bound_it = attributes.inputs.find(DiagonalBandMask_attributes::input_names::ShiftRightBound);
+ return ((shift_right_bound_it) != attributes.inputs.end() && shift_right_bound_it->second != nullptr);
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ }
+#endif
+};
+
+// Fallback implementation of diagonal band mask node that represents the mask as a series of pointwise operations.
+// This is used for cuDNN versions before v9.21.0.
+// Note: the individual masks can be constructed in different ways as well that yield functionally
+// correct results. However, for performance reasons in the cuDNN backend they are organized as they are. Be
+// cautious of performance when editing.
+class CompositeDiagonalBandMaskNode : public DiagonalBandMaskNodeBase<CompositeDiagonalBandMaskNode> {
+ public:
+ CompositeDiagonalBandMaskNode(DiagonalBandMask_attributes&& attributes_, detail::Context const& context)
+ : DiagonalBandMaskNodeBase(std::move(attributes_), context) {}
+
+ Type
+ getType() override final {
+ return Type::COMPOSITE;
+ }
+
+ error_t
+ pre_validate_node_extra() const override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating CompositeDiagonalBandMaskNode " << attributes.name);
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(has_cu_seq_len_q() || has_cu_seq_len_kv(),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "CompositeDiagonalBandMaskNode does not support CU_SEQ_LEN_Q or CU_SEQ_LEN_KV");
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ expand_node() override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for CompositeDiagonalBandMaskNode "
+ << attributes.name);
+
+ attributes.fill_from_context(context);
+
+ auto attention_score = attributes.inputs[DiagonalBandMask_attributes::input_names::X];
+ auto negative_inf_value = attributes.inputs[DiagonalBandMask_attributes::input_names::B];
+
+ auto row_index = pointwise(
+ attention_score,
+ Pointwise_attributes().set_name("gen_row_index").set_mode(PointwiseMode_t::GEN_INDEX).set_axis(2));
+ row_index->set_data_type(DataType_t::INT32);
+ auto col_index = pointwise(
+ attention_score,
+ Pointwise_attributes().set_name("gen_col_index").set_mode(PointwiseMode_t::GEN_INDEX).set_axis(3));
+ col_index->set_data_type(DataType_t::INT32);
+
+ std::shared_ptr<Tensor_attributes> swa_comparison_output;
+
+ if (has_right_bound()) {
+ std::string row_index_name = "row";
+
+ if (has_seq_len_kv()) {
+ auto seq_len_kv = attributes.inputs[DiagonalBandMask_attributes::input_names::SEQ_LEN_KV];
+ row_index = pointwise(row_index,
+ seq_len_kv,
+ Pointwise_attributes()
+ .set_name(row_index_name += "+skv")
+ .set_mode(PointwiseMode_t::ADD)
+ .set_compute_data_type(DataType_t::INT32)
+ .set_axis(3));
+ row_index->set_data_type(DataType_t::INT32);
+ }
+
+ if (has_seq_len_q()) {
+ auto seq_len_q = attributes.inputs[DiagonalBandMask_attributes::input_names::SEQ_LEN_Q];
+ row_index = pointwise(row_index,
+ seq_len_q,
+ Pointwise_attributes()
+ .set_name(row_index_name += "-sq")
+ .set_mode(PointwiseMode_t::SUB)
+ .set_compute_data_type(DataType_t::INT32)
+ .set_axis(3));
+ row_index->set_data_type(DataType_t::INT32);
+ }
+
+ if (has_shift_right_bound()) {
+ auto shift_right_bound = attributes.inputs[DiagonalBandMask_attributes::input_names::ShiftRightBound];
+ // Use shift_right_bound's data type
+ row_index = pointwise(row_index,
+ shift_right_bound,
+ Pointwise_attributes()
+ .set_name(row_index_name += "+window")
+ .set_mode(PointwiseMode_t::ADD)
+ .set_compute_data_type(shift_right_bound->get_data_type())
+ .set_axis(3));
+ row_index->set_data_type(shift_right_bound->get_data_type());
+ }
+
+ swa_comparison_output = pointwise(row_index,
+ col_index,
+ Pointwise_attributes()
+ .set_name(row_index_name += ">col")
+ .set_mode(attributes.comparison_mode)
+ .set_compute_data_type(DataType_t::BOOLEAN));
+ swa_comparison_output->set_data_type(DataType_t::BOOLEAN);
+ }
+
+ // Set the left bound
+ else if (has_left_bound()) {
+ auto left_bound = attributes.inputs[DiagonalBandMask_attributes::input_names::LeftBound];
+ std::string col_index_name = "col";
+
+ // Use left_bound's data type
+ col_index = pointwise(col_index,
+ left_bound,
+ Pointwise_attributes()
+ .set_name(col_index_name += "+window")
+ .set_mode(PointwiseMode_t::ADD)
+ .set_compute_data_type(left_bound->get_data_type())
+ .set_axis(3));
+ col_index->set_data_type(left_bound->get_data_type());
+
+ if (has_seq_len_kv()) {
+ auto seq_len_kv = attributes.inputs[DiagonalBandMask_attributes::input_names::SEQ_LEN_KV];
+ col_index = pointwise(col_index,
+ seq_len_kv,
+ Pointwise_attributes()
+ .set_name(col_index_name += "-skv")
+ .set_mode(PointwiseMode_t::SUB)
+ .set_compute_data_type(DataType_t::INT32)
+ .set_axis(3));
+ col_index->set_data_type(DataType_t::INT32);
+ }
+
+ if (has_seq_len_q()) {
+ auto seq_len_q = attributes.inputs[DiagonalBandMask_attributes::input_names::SEQ_LEN_Q];
+ col_index = pointwise(col_index,
+ seq_len_q,
+ Pointwise_attributes()
+ .set_name(col_index_name += "+sq")
+ .set_mode(PointwiseMode_t::ADD)
+ .set_compute_data_type(DataType_t::INT32)
+ .set_axis(3));
+ col_index->set_data_type(DataType_t::INT32);
+ }
+
+ swa_comparison_output = pointwise(col_index,
+ row_index,
+ Pointwise_attributes()
+ .set_name(col_index_name += ">row")
+ .set_mode(attributes.comparison_mode)
+ .set_compute_data_type(DataType_t::BOOLEAN));
+ swa_comparison_output->set_data_type(DataType_t::BOOLEAN);
+ } else {
+ // This should not be reached (since one of has_left_bound() or has_right_bound() is always true by
+ // construction), just here for completeness.
+ return {error_code_t::INVALID_VALUE,
+ "CompositeDiagonalBandMaskNode must have a left bound or a right bound"};
+ }
+
+ // Special non-functional-style call. Needed because output already created and provided to user.
+ pointwise(attention_score,
+ negative_inf_value,
+ swa_comparison_output,
+ Pointwise_attributes().set_name("binary_select").set_mode(PointwiseMode_t::BINARY_SELECT),
+ attributes.outputs[DiagonalBandMask_attributes::output_names::Y]);
+
+ return {error_code_t::OK, ""};
+ }
+};
+
+// Newer implementation of diagonal band mask node that represents the mask as a single operation.
+// This is used for cuDNN versions v9.21.0 and later.
+class UnifiedDiagonalBandMaskNode : public DiagonalBandMaskNodeBase<UnifiedDiagonalBandMaskNode> {
+ public:
+ UnifiedDiagonalBandMaskNode(DiagonalBandMask_attributes&& attributes_, detail::Context const& context)
+ : DiagonalBandMaskNodeBase(std::move(attributes_), context) {}
+
+ Type
+ getType() override final {
+ return Type::DIAGONAL_BAND_MASK;
+ }
+
+ error_t
+ create_cudnn_operations(
+ std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations,
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations,
+ managed_backend_descriptor_t& raw_operations,
+ std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final {
+ getLogger() << "[cudnn_frontend] INFO: " << "Building UnifiedDiagonalBandMaskNode operations "
+ << attributes.name << std::endl;
+ auto cudnn_ver_error =
+ error_t{error_code_t::GRAPH_NOT_SUPPORTED, "UnifiedDiagonalBandMaskNode requires cuDNN v9.21.0"};
+
+#if (CUDNN_VERSION >= 92100)
+ NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(92100, cudnn_ver_error);
+ CUDNN_FRONTEND_UNUSED(operations);
+ auto diagonal_band_mask_operation = make_shared_backend_pointer(
+ (cudnnBackendDescriptorType_t)CUDNN_BACKEND_OPERATION_DIAGONAL_BAND_MASK_DESCRIPTOR);
+
+ auto X = attributes.inputs.find(DiagonalBandMask_attributes::input_names::X)->second;
+ auto backend_x = tensors[X->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(diagonal_band_mask_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_XDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_x));
+
+ auto B = attributes.inputs.find(DiagonalBandMask_attributes::input_names::B)->second;
+ auto backend_b = tensors[B->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(diagonal_band_mask_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_BDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_b));
+
+ if (has_seq_len_q()) {
+ auto seq_len_Q = attributes.inputs.find(DiagonalBandMask_attributes::input_names::SEQ_LEN_Q)->second;
+ auto backend_seq_len_Q = tensors[seq_len_Q->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(diagonal_band_mask_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_SEQ_LEN_QDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_seq_len_Q));
+ }
+
+ if (has_seq_len_kv()) {
+ auto seq_len_KV = attributes.inputs.find(DiagonalBandMask_attributes::input_names::SEQ_LEN_KV)->second;
+ auto backend_seq_len_KV = tensors[seq_len_KV->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(diagonal_band_mask_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_SEQ_LEN_KVDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_seq_len_KV));
+ }
+
+ if (has_cu_seq_len_q() || has_cu_seq_len_kv()) {
+ auto cu_seq_len_cudnn_ver_error =
+ error_t{error_code_t::GRAPH_NOT_SUPPORTED,
+ "Cumulative sequence length tensors in unified DiagonalBandMask node require cuDNN 9.24.0"};
+#if (CUDNN_VERSION >= 92400)
+ NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(92400, cu_seq_len_cudnn_ver_error);
+
+ if (has_cu_seq_len_q()) {
+ auto cu_seq_len_Q =
+ attributes.inputs.find(DiagonalBandMask_attributes::input_names::CU_SEQ_LEN_Q)->second;
+ auto backend_cu_seq_len_Q = tensors[cu_seq_len_Q->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(diagonal_band_mask_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_CU_SEQ_LEN_QDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_cu_seq_len_Q));
+ }
+
+ if (has_cu_seq_len_kv()) {
+ auto cu_seq_len_KV =
+ attributes.inputs.find(DiagonalBandMask_attributes::input_names::CU_SEQ_LEN_KV)->second;
+ auto backend_cu_seq_len_KV = tensors[cu_seq_len_KV->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(
+ detail::set_attribute(diagonal_band_mask_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_CU_SEQ_LEN_KVDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_cu_seq_len_KV));
+ }
+#else
+ return cu_seq_len_cudnn_ver_error;
+#endif
+ }
+
+ if (has_left_bound()) {
+ auto left_bound = attributes.inputs.find(DiagonalBandMask_attributes::input_names::LeftBound)->second;
+ auto backend_left_bound = tensors[left_bound->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(diagonal_band_mask_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_LEFT_BOUND_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_left_bound));
+ }
+
+ if (has_shift_right_bound()) {
+ auto shift_right_bound =
+ attributes.inputs.find(DiagonalBandMask_attributes::input_names::ShiftRightBound)->second;
+ auto backend_shift_right_bound =
+ tensors[shift_right_bound->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(
+ detail::set_attribute(diagonal_band_mask_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_SHIFT_RIGHT_BOUND_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_shift_right_bound));
+ }
+
+ auto Y = attributes.outputs.find(DiagonalBandMask_attributes::output_names::Y)->second;
+ auto backend_y = tensors[Y->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(diagonal_band_mask_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_YDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_y));
+
+ cudnnPointwiseMode_t cudnn_pointwise_mode;
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.comparison_mode, cudnn_pointwise_mode));
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(diagonal_band_mask_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_COMPARISON_MODE,
+ CUDNN_TYPE_POINTWISE_MODE,
+ 1,
+ &cudnn_pointwise_mode));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(diagonal_band_mask_operation->get_backend_descriptor()));
+
+ raw_operations.push_back(diagonal_band_mask_operation);
+
+ auto const& non_virtual_uids = attributes.get_non_virtual_uids();
+ uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end());
+ return {error_code_t::OK, ""};
+#else
+ CUDNN_FRONTEND_UNUSED(uids_involved_in_operations);
+ CUDNN_FRONTEND_UNUSED(operations);
+ CUDNN_FRONTEND_UNUSED(raw_operations);
+ CUDNN_FRONTEND_UNUSED(tensors);
+ return cudnn_ver_error;
+#endif // CUDNN_VERSION >= 92100
+ }
+};
+
+// Factory function to create a diagonal band mask node.
+// This function will create the appropriate node based on the cuDNN version.
+inline std::shared_ptr<Tensor_attributes>
+INode::diagonal_band_mask(std::shared_ptr<Tensor_attributes> x,
+ std::shared_ptr<Tensor_attributes> b,
+ std::shared_ptr<Tensor_attributes> seq_len_q,
+ std::shared_ptr<Tensor_attributes> seq_len_kv,
+ std::shared_ptr<Tensor_attributes> cu_seq_len_q,
+ std::shared_ptr<Tensor_attributes> cu_seq_len_kv,
+ std::shared_ptr<Tensor_attributes> left_bound,
+ std::shared_ptr<Tensor_attributes> shift_right_bound,
+ DiagonalBandMask_attributes attributes) {
+ attributes.inputs[DiagonalBandMask_attributes::input_names::X] = x;
+ attributes.inputs[DiagonalBandMask_attributes::input_names::B] = b;
+ if (seq_len_q) {
+ attributes.inputs[DiagonalBandMask_attributes::input_names::SEQ_LEN_Q] = seq_len_q;
+ }
+ if (seq_len_kv) {
+ attributes.inputs[DiagonalBandMask_attributes::input_names::SEQ_LEN_KV] = seq_len_kv;
+ }
+ if (cu_seq_len_q) {
+ attributes.inputs[DiagonalBandMask_attributes::input_names::CU_SEQ_LEN_Q] = cu_seq_len_q;
+ }
+ if (cu_seq_len_kv) {
+ attributes.inputs[DiagonalBandMask_attributes::input_names::CU_SEQ_LEN_KV] = cu_seq_len_kv;
+ }
+ if (left_bound) {
+ attributes.inputs[DiagonalBandMask_attributes::input_names::LeftBound] = left_bound;
+ }
+ if (shift_right_bound) {
+ attributes.inputs[DiagonalBandMask_attributes::input_names::ShiftRightBound] = shift_right_bound;
+ }
+ auto y = attributes.outputs[DiagonalBandMask_attributes::output_names::Y] = output_tensor(attributes.name + "::Y");
+
+ // Newer versions of cuDNN can represent the diagonal band mask as a single operation.
+ if (std::min(detail::get_compiled_version(), detail::get_backend_version()) >= 92100) {
+ sub_nodes.emplace_back(std::make_unique<UnifiedDiagonalBandMaskNode>(std::move(attributes), context));
+ } else {
+ sub_nodes.emplace_back(std::make_unique<CompositeDiagonalBandMaskNode>(std::move(attributes), context));
+ }
+ return y;
+}
+
+} // namespace cudnn_frontend::graph \ No newline at end of file
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/dln.h b/deps/cudnn-frontend/include/cudnn_frontend/node/dln.h
new file mode 100644
index 0000000..ad7ad1f
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/node/dln.h
@@ -0,0 +1,226 @@
+#pragma once
+
+#include "../graph_helpers.h"
+#include "../node_interface.h"
+
+namespace cudnn_frontend {
+
+namespace graph {
+
+class DLNNode : public NodeCRTP<DLNNode> {
+ public:
+ Layernorm_backward_attributes attributes;
+
+ DLNNode(Layernorm_backward_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP(context), attributes(std::move(attributes_)) {}
+
+ Type
+ getType() override final {
+ return Type::DLN;
+ }
+ error_t
+ infer_properties_node() override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for DLN node " << attributes.name);
+
+ if (detail::get_backend_version() < 8906) {
+ attributes.inputs[Layernorm_backward_attributes::input_names::EPSILON] =
+ std::make_shared<Tensor_attributes>(0.0f);
+ }
+
+ attributes.fill_from_context(context);
+
+ // TODO: Only inferencing from X works today.
+ auto X = attributes.inputs[Layernorm_backward_attributes::input_names::X];
+ auto const x_tensor_dim = X->get_dim();
+
+ auto DY = attributes.inputs[Layernorm_backward_attributes::input_names::DY];
+ auto dy_tensor_dim = DY->get_dim();
+
+ // Only infer dims and strides if user did not set them
+ if (dy_tensor_dim.empty()) {
+ dy_tensor_dim.resize(x_tensor_dim.size());
+ DY->set_dim(x_tensor_dim);
+ }
+ if (DY->get_stride().empty()) {
+ auto const& DY_dim = DY->get_dim();
+ // Default to NHWC
+ auto const& stride_order = detail::generate_NHWC_stride_order(DY_dim.size());
+ DY->set_stride(detail::generate_stride(DY_dim, stride_order));
+ }
+
+ auto DX = attributes.outputs[Layernorm_backward_attributes::output_names::DX];
+ auto dx_tensor_dim = DX->get_dim();
+ // Only infer dims and strides if user did not set them
+ if (dx_tensor_dim.empty()) {
+ dx_tensor_dim.resize(x_tensor_dim.size());
+ DX->set_dim(x_tensor_dim);
+ }
+ if (DX->get_stride().empty()) {
+ auto const& DX_dim = DX->get_dim();
+ // Default to NHWC
+ auto const& stride_order = detail::generate_NHWC_stride_order(DX_dim.size());
+ DX->set_stride(detail::generate_stride(DX_dim, stride_order));
+ }
+
+ auto scale_bias_dim = X->get_dim();
+ scale_bias_dim[0] = 1;
+
+ // Set channel length tensors
+ auto infer_scale_bias_tensors = [&scale_bias_dim](std::shared_ptr<Tensor_attributes>& T) {
+ auto tensor_dim = T->get_dim();
+ // Only infer dims and strides if user did not set them
+ if (tensor_dim.empty()) {
+ T->set_dim(scale_bias_dim);
+ }
+ if (T->get_stride().empty()) {
+ auto const& T_dim = T->get_dim();
+ // Default to NHWC
+ auto const& stride_order = detail::generate_NHWC_stride_order(T_dim.size());
+ T->set_stride(detail::generate_stride(T_dim, stride_order));
+ }
+ };
+
+ infer_scale_bias_tensors(attributes.outputs[Layernorm_backward_attributes::output_names::DSCALE]);
+ infer_scale_bias_tensors(attributes.outputs[Layernorm_backward_attributes::output_names::DBIAS]);
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ create_cudnn_operations(
+ std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations,
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations,
+ managed_backend_descriptor_t& raw_operations,
+ std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final {
+ CUDNN_FRONTEND_UNUSED(raw_operations);
+ CUDNN_FE_LOG_LABEL("INFO: " << "Building DLNNode operations " << attributes.name << " ");
+
+ // Create operation by directly calling cuDNN backend API
+ Operation_v8 dln_operation;
+
+ _CUDNN_CHECK_CUDNN_ERROR(
+ dln_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_NORM_BACKWARD_DESCRIPTOR));
+
+ // Set norm mode to LAYER_NORM
+ cudnnBackendNormMode_t cudnn_norm_mode;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormMode_t::LAYER_NORM, cudnn_norm_mode));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dln_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_MODE,
+ CUDNN_TYPE_NORM_MODE,
+ 1,
+ &cudnn_norm_mode));
+
+ // Set input tensor X
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Layernorm_backward_attributes::input_names::X);
+ auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dln_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_XDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &x_desc));
+
+ // Set DY tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(DY, Layernorm_backward_attributes::input_names::DY);
+ auto dy_desc = tensors.at(DY->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dln_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_DYDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &dy_desc));
+
+ // Set scale tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SCALE, Layernorm_backward_attributes::input_names::SCALE);
+ auto scale_desc = tensors.at(SCALE->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dln_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_SCALE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &scale_desc));
+
+ // Set mean and inv_variance tensors
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(MEAN, Layernorm_backward_attributes::input_names::MEAN);
+ auto mean_desc = tensors.at(MEAN->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dln_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_MEAN_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &mean_desc));
+
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(INV_VARIANCE,
+ Layernorm_backward_attributes::input_names::INV_VARIANCE);
+ auto inv_var_desc = tensors.at(INV_VARIANCE->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dln_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_INV_VARIANCE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &inv_var_desc));
+
+ // Set DSCALE and DBIAS output tensors
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DSCALE, Layernorm_backward_attributes::output_names::DSCALE);
+ auto dscale_desc = tensors.at(DSCALE->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dln_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_DSCALE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &dscale_desc));
+
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DBIAS, Layernorm_backward_attributes::output_names::DBIAS);
+ auto dbias_desc = tensors.at(DBIAS->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dln_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_DBIAS_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &dbias_desc));
+
+ // Set DX output tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DX, Layernorm_backward_attributes::output_names::DX);
+ auto dx_desc = tensors.at(DX->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dln_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_DXDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &dx_desc));
+
+ // Set epsilon tensor for older backend versions
+ if (detail::get_backend_version() < 8906) {
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(EPSILON, Layernorm_backward_attributes::input_names::EPSILON);
+ auto epsilon_desc = tensors.at(EPSILON->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dln_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_EPSILON_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &epsilon_desc));
+ }
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(dln_operation.get_raw_desc()));
+
+ operations.push_back(std::make_shared<Operation_v8>(std::move(dln_operation)));
+
+ auto const& non_virtual_uids = attributes.get_non_virtual_uids();
+ uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end());
+ return {error_code_t::OK, ""};
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ j.update(R"( {"tag": "LAYER_NORM_BPROP"})"_json);
+ }
+#endif
+};
+
+} // namespace graph
+
+} // namespace cudnn_frontend
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/genstats.h b/deps/cudnn-frontend/include/cudnn_frontend/node/genstats.h
new file mode 100644
index 0000000..8f91897
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/node/genstats.h
@@ -0,0 +1,147 @@
+#pragma once
+
+#include "../graph_helpers.h"
+#include "../node_interface.h"
+
+namespace cudnn_frontend {
+
+namespace graph {
+
+class GenstatsNode : public NodeCRTP<GenstatsNode> {
+ public:
+ Genstats_attributes attributes;
+
+ GenstatsNode(Genstats_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP(context), attributes(std::move(attributes_)) {}
+
+ Type
+ getType() override final {
+ return Type::GENSTATS;
+ }
+
+ error_t
+ infer_properties_node() override final {
+ attributes.fill_from_context(context);
+
+ // Only inferrencing from X works today.
+ auto X = attributes.inputs[Genstats_attributes::input_names::X];
+ auto SUM = attributes.outputs[Genstats_attributes::output_names::SUM];
+ auto SQ_SUM = attributes.outputs[Genstats_attributes::output_names::SQ_SUM];
+
+ auto const x_tensor_dim = X->get_dim();
+ auto sum_tensor_dim = SUM->get_dim();
+ auto sq_sum_tensor_dim = SQ_SUM->get_dim();
+
+ // Only infer dims and strides if user did not set them
+ if (sum_tensor_dim.empty()) {
+ sum_tensor_dim.resize(x_tensor_dim.size(), 1);
+ sum_tensor_dim[1] = x_tensor_dim[1];
+ SUM->set_dim(sum_tensor_dim);
+ }
+ if (SUM->get_stride().empty()) {
+ auto const& SUM_dim = SUM->get_dim();
+ // Default to NHWC
+ auto const& stride_order = detail::generate_NHWC_stride_order(SUM_dim.size());
+ SUM->set_stride(detail::generate_stride(SUM_dim, stride_order));
+ }
+
+ // Only infer dims and strides if user did not set them
+ if (sq_sum_tensor_dim.empty()) {
+ sq_sum_tensor_dim.resize(x_tensor_dim.size(), 1);
+ sq_sum_tensor_dim[1] = x_tensor_dim[1];
+ SQ_SUM->set_dim(sq_sum_tensor_dim);
+ }
+ if (SQ_SUM->get_stride().empty()) {
+ auto const& SQ_SUM_dim = SQ_SUM->get_dim();
+ // Default to NHWC
+ auto const& stride_order = detail::generate_NHWC_stride_order(SQ_SUM_dim.size());
+ SQ_SUM->set_stride(detail::generate_stride(SQ_SUM_dim, stride_order));
+ }
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ create_cudnn_operations(
+ std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations,
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations,
+ managed_backend_descriptor_t& raw_operations,
+ std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final {
+ CUDNN_FRONTEND_UNUSED(raw_operations);
+ CUDNN_FE_LOG_LABEL("INFO: " << "Building GenstatsNode operations " << attributes.name << " ");
+
+ // Create operation by directly calling cuDNN backend API
+ Operation_v8 genstats_operation;
+
+ _CUDNN_CHECK_CUDNN_ERROR(
+ genstats_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_GEN_STATS_DESCRIPTOR));
+
+ // Set input tensor X
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Genstats_attributes::input_names::X);
+ auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(genstats_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_GENSTATS_XDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &x_desc));
+
+ // Set gen stats mode
+ cudnnGenStatsMode_t genstats_mode = CUDNN_GENSTATS_SUM_SQSUM;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(genstats_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_GENSTATS_MODE,
+ CUDNN_TYPE_GENSTATS_MODE,
+ 1,
+ &genstats_mode));
+
+ // Set math precision based on X tensor data type
+ cudnnDataType_t math_prec = static_cast<cudnnDataType_t>(tensors.at(X->second->get_uid())->getDataType());
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(genstats_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_GENSTATS_MATH_PREC,
+ CUDNN_TYPE_DATA_TYPE,
+ 1,
+ &math_prec));
+
+ // Set SUM output tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(SUM, Genstats_attributes::output_names::SUM);
+ auto sum_desc = tensors.at(SUM->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(genstats_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_GENSTATS_SUMDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &sum_desc));
+
+ // Set SQ_SUM output tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(SQ_SUM, Genstats_attributes::output_names::SQ_SUM);
+ auto sq_sum_desc = tensors.at(SQ_SUM->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(genstats_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_GENSTATS_SQSUMDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &sq_sum_desc));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(genstats_operation.get_raw_desc()));
+
+ operations.push_back(std::make_shared<Operation_v8>(std::move(genstats_operation)));
+
+ auto const& non_virtual_uids = attributes.get_non_virtual_uids();
+ uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end());
+ return {error_code_t::OK, ""};
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ j.update(R"( {"tag": "GENSTATS"})"_json);
+ }
+#endif
+};
+
+} // namespace graph
+
+} // namespace cudnn_frontend \ No newline at end of file
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/instancenorm.h b/deps/cudnn-frontend/include/cudnn_frontend/node/instancenorm.h
new file mode 100644
index 0000000..1b71f4a
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/node/instancenorm.h
@@ -0,0 +1,414 @@
+#pragma once
+
+#include "../graph_helpers.h"
+#include "../node_interface.h"
+
+namespace cudnn_frontend {
+
+namespace graph {
+class InstanceNormNode : public NodeCRTP<InstanceNormNode> {
+ public:
+ Instancenorm_attributes attributes;
+
+ InstanceNormNode(Instancenorm_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP(context), attributes(std::move(attributes_)) {}
+
+ Type
+ getType() override final {
+ return Type::INSTANCENORM;
+ }
+
+ error_t
+ infer_properties_node() override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for instancenorm node " << attributes.name);
+
+ attributes.fill_from_context(context);
+
+ auto X = attributes.inputs[Instancenorm_attributes::input_names::X];
+ auto Y = attributes.outputs[Instancenorm_attributes::output_names::Y];
+
+ // Only infer dims and strides if user did not set them
+ if (Y->get_dim().empty()) {
+ Y->set_dim(X->get_dim());
+ }
+ if (Y->get_stride().empty()) {
+ Y->set_stride(X->get_stride());
+ }
+
+ // mean inv_var dim is n,c,1,1
+ auto stats_dim = X->get_dim();
+ for (size_t i = 2; i < stats_dim.size(); i++) {
+ stats_dim[i] = 1;
+ }
+
+ if (attributes.forward_phase == NormFwdPhase_t::TRAINING) {
+ auto mean = attributes.outputs[Instancenorm_attributes::output_names::MEAN];
+ if (mean->get_dim().empty()) {
+ mean->set_dim(stats_dim);
+ }
+ if (mean->get_stride().empty()) {
+ auto const& mean_dim = mean->get_dim();
+ // Default to NHWC
+ auto const& stride_order = detail::generate_NHWC_stride_order(mean_dim.size());
+ mean->set_stride(detail::generate_stride(mean_dim, stride_order));
+ }
+
+ auto inv_var = attributes.outputs[Instancenorm_attributes::output_names::INV_VARIANCE];
+ if (inv_var->get_dim().empty()) {
+ inv_var->set_dim(stats_dim);
+ }
+ if (inv_var->get_stride().empty()) {
+ auto const& inv_var_dim = inv_var->get_dim();
+ // Default to NHWC
+ auto const& stride_order = detail::generate_NHWC_stride_order(inv_var_dim.size());
+ inv_var->set_stride(detail::generate_stride(inv_var_dim, stride_order));
+ }
+ }
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ pre_validate_node() const override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating InstanceNormNode " << attributes.name);
+
+ // Norm forward phase should be set
+ RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.forward_phase == NormFwdPhase_t::NOT_SET,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "Forward phase not set of instancenorm node.");
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ create_cudnn_operations(
+ std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations,
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations,
+ managed_backend_descriptor_t& raw_operations,
+ std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final {
+ CUDNN_FRONTEND_UNUSED(raw_operations);
+ CUDNN_FE_LOG_LABEL("INFO: Building InstanceNormNode operations " << attributes.name << " ");
+
+ // Create operation by directly calling cuDNN backend API
+ Operation_v8 instancenorm_operation;
+
+ _CUDNN_CHECK_CUDNN_ERROR(
+ instancenorm_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_NORM_FORWARD_DESCRIPTOR));
+
+ // Set norm mode to INSTANCE_NORM
+ cudnnBackendNormMode_t cudnn_norm_mode;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormMode_t::INSTANCE_NORM, cudnn_norm_mode));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(instancenorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_MODE,
+ CUDNN_TYPE_NORM_MODE,
+ 1,
+ &cudnn_norm_mode));
+
+ // Set forward phase
+ cudnnBackendNormFwdPhase_t cudnn_norm_fwd_phase;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.forward_phase, cudnn_norm_fwd_phase));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(instancenorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_PHASE,
+ CUDNN_TYPE_NORM_FWD_PHASE,
+ 1,
+ &cudnn_norm_fwd_phase));
+
+ // Set input tensor X
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Instancenorm_attributes::input_names::X);
+ auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(instancenorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_XDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &x_desc));
+
+ // Set scale and bias tensors
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SCALE, Instancenorm_attributes::input_names::SCALE);
+ auto scale_desc = tensors.at(SCALE->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(instancenorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_SCALE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &scale_desc));
+
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(BIAS, Instancenorm_attributes::input_names::BIAS);
+ auto bias_desc = tensors.at(BIAS->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(instancenorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_BIAS_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &bias_desc));
+
+ // Set epsilon tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(EPSILON, Instancenorm_attributes::input_names::EPSILON);
+ auto epsilon_desc = tensors.at(EPSILON->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(instancenorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_EPSILON_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &epsilon_desc));
+
+ // Set output tensor Y
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(Y, Instancenorm_attributes::output_names::Y);
+ auto y_desc = tensors.at(Y->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(instancenorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_YDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &y_desc));
+
+ // Set mean and inv_variance for training phase
+ if (attributes.forward_phase == NormFwdPhase_t::TRAINING) {
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(MEAN, Instancenorm_attributes::output_names::MEAN);
+ auto mean_desc = tensors.at(MEAN->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(instancenorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_MEAN_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &mean_desc));
+
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(INV_VARIANCE,
+ Instancenorm_attributes::output_names::INV_VARIANCE);
+ auto inv_var_desc = tensors.at(INV_VARIANCE->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(instancenorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_INV_VARIANCE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &inv_var_desc));
+ }
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(instancenorm_operation.get_raw_desc()));
+
+ operations.push_back(std::make_shared<Operation_v8>(std::move(instancenorm_operation)));
+
+ auto const& non_virtual_uids = attributes.get_non_virtual_uids();
+ uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end());
+ return {error_code_t::OK, ""};
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ j.update(R"( {"tag": "INSTANCE_NORM"})"_json);
+ }
+#endif
+};
+
+class DINNode : public NodeCRTP<DINNode> {
+ public:
+ Instancenorm_backward_attributes attributes;
+
+ DINNode(Instancenorm_backward_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP(context), attributes(std::move(attributes_)) {}
+
+ Type
+ getType() override final {
+ return Type::DIN;
+ }
+
+ error_t
+ infer_properties_node() override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for DIN node " << attributes.name);
+
+ attributes.fill_from_context(context);
+
+ // TODO: Only inferencing from X works today.
+ auto X = attributes.inputs[Instancenorm_backward_attributes::input_names::X];
+ auto const x_tensor_dim = X->get_dim();
+
+ auto DY = attributes.inputs[Instancenorm_backward_attributes::input_names::DY];
+ auto dy_tensor_dim = DY->get_dim();
+
+ // Only infer dims and strides if user did not set them
+ if (dy_tensor_dim.empty()) {
+ dy_tensor_dim.resize(x_tensor_dim.size());
+ DY->set_dim(x_tensor_dim);
+ }
+ if (DY->get_stride().empty()) {
+ auto const& DY_dim = DY->get_dim();
+ // Default to NHWC
+ auto const& stride_order = detail::generate_NHWC_stride_order(DY_dim.size());
+ DY->set_stride(detail::generate_stride(DY_dim, stride_order));
+ }
+
+ auto DX = attributes.outputs[Instancenorm_backward_attributes::output_names::DX];
+ auto dx_tensor_dim = DX->get_dim();
+ // Only infer dims and strides if user did not set them
+ if (dx_tensor_dim.empty()) {
+ dx_tensor_dim.resize(x_tensor_dim.size());
+ DX->set_dim(x_tensor_dim);
+ }
+ if (DX->get_stride().empty()) {
+ auto const& DX_dim = DX->get_dim();
+ // Default to NHWC
+ auto const& stride_order = detail::generate_NHWC_stride_order(DX_dim.size());
+ DX->set_stride(detail::generate_stride(DX_dim, stride_order));
+ }
+
+ // scale_bias dim is 1,c,1,1
+ // mean inv_var dim is n,c,1,1
+ auto scale_bias_dim = X->get_dim();
+ for (size_t i = 0; i < scale_bias_dim.size(); i++) {
+ if (i != 1) {
+ scale_bias_dim[i] = 1;
+ }
+ }
+
+ // Set channel length tensors
+ auto infer_scale_bias_tensors = [&scale_bias_dim](std::shared_ptr<Tensor_attributes>& T) {
+ auto tensor_dim = T->get_dim();
+ // Only infer dims and strides if user did not set them
+ if (tensor_dim.empty()) {
+ T->set_dim(scale_bias_dim);
+ }
+ if (T->get_stride().empty()) {
+ auto const& T_dim = T->get_dim();
+ // Default to NHWC
+ auto const& stride_order = detail::generate_NHWC_stride_order(T_dim.size());
+ T->set_stride(detail::generate_stride(T_dim, stride_order));
+ }
+ };
+
+ infer_scale_bias_tensors(attributes.outputs[Instancenorm_backward_attributes::output_names::DSCALE]);
+ infer_scale_bias_tensors(attributes.outputs[Instancenorm_backward_attributes::output_names::DBIAS]);
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ create_cudnn_operations(
+ std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations,
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations,
+ managed_backend_descriptor_t& raw_operations,
+ std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final {
+ CUDNN_FRONTEND_UNUSED(raw_operations);
+ CUDNN_FE_LOG_LABEL("INFO: Building DINNode operations " << attributes.name << " ");
+
+ // Create operation by directly calling cuDNN backend API
+ Operation_v8 din_operation;
+
+ _CUDNN_CHECK_CUDNN_ERROR(
+ din_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_NORM_BACKWARD_DESCRIPTOR));
+
+ // Set norm mode to INSTANCE_NORM
+ cudnnBackendNormMode_t cudnn_norm_mode;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormMode_t::INSTANCE_NORM, cudnn_norm_mode));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(din_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_MODE,
+ CUDNN_TYPE_NORM_MODE,
+ 1,
+ &cudnn_norm_mode));
+
+ // Set input tensor X
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Instancenorm_backward_attributes::input_names::X);
+ auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(din_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_XDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &x_desc));
+
+ // Set DY tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(DY, Instancenorm_backward_attributes::input_names::DY);
+ auto dy_desc = tensors.at(DY->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(din_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_DYDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &dy_desc));
+
+ // Set scale tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SCALE, Instancenorm_backward_attributes::input_names::SCALE);
+ auto scale_desc = tensors.at(SCALE->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(din_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_SCALE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &scale_desc));
+
+ // Set mean and inv_variance tensors
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(MEAN, Instancenorm_backward_attributes::input_names::MEAN);
+ auto mean_desc = tensors.at(MEAN->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(din_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_MEAN_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &mean_desc));
+
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(INV_VARIANCE,
+ Instancenorm_backward_attributes::input_names::INV_VARIANCE);
+ auto inv_var_desc = tensors.at(INV_VARIANCE->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(din_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_INV_VARIANCE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &inv_var_desc));
+
+ // Set DSCALE and DBIAS output tensors
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DSCALE, Instancenorm_backward_attributes::output_names::DSCALE);
+ auto dscale_desc = tensors.at(DSCALE->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(din_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_DSCALE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &dscale_desc));
+
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DBIAS, Instancenorm_backward_attributes::output_names::DBIAS);
+ auto dbias_desc = tensors.at(DBIAS->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(din_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_DBIAS_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &dbias_desc));
+
+ // Set DX output tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DX, Instancenorm_backward_attributes::output_names::DX);
+ auto dx_desc = tensors.at(DX->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(din_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_DXDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &dx_desc));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(din_operation.get_raw_desc()));
+
+ operations.push_back(std::make_shared<Operation_v8>(std::move(din_operation)));
+
+ auto const& non_virtual_uids = attributes.get_non_virtual_uids();
+ uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end());
+ return {error_code_t::OK, ""};
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ j.update(R"( {"tag": "INSTANCE_NORM_BPROP"})"_json);
+ }
+#endif
+};
+
+} // namespace graph
+
+} // namespace cudnn_frontend \ No newline at end of file
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/layernorm.h b/deps/cudnn-frontend/include/cudnn_frontend/node/layernorm.h
new file mode 100644
index 0000000..46420aa
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/node/layernorm.h
@@ -0,0 +1,259 @@
+#pragma once
+
+#include "../graph_helpers.h"
+#include "../node_interface.h"
+
+namespace cudnn_frontend {
+
+namespace graph {
+class LayerNormNode : public NodeCRTP<LayerNormNode> {
+ public:
+ Layernorm_attributes attributes;
+
+ LayerNormNode(Layernorm_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP(context), attributes(std::move(attributes_)) {}
+
+ Type
+ getType() override final {
+ return Type::LAYERNORM;
+ }
+
+ error_t
+ infer_properties_node() override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for layernorm node " << attributes.name);
+
+ attributes.fill_from_context(context);
+
+ auto X = attributes.inputs[Layernorm_attributes::input_names::X];
+ auto Y = attributes.outputs[Layernorm_attributes::output_names::Y];
+
+ // Only infer dims and strides if user did not set them
+ if (Y->get_dim().empty()) {
+ Y->set_dim(X->get_dim());
+ }
+ if (Y->get_stride().empty()) {
+ Y->set_stride(X->get_stride());
+ }
+
+ // scale_bias dim is 1,c,h,w
+ auto scale_bias_dim = X->get_dim();
+ scale_bias_dim[0] = 1;
+
+ auto scale = attributes.inputs[Layernorm_attributes::input_names::SCALE];
+ // Only infer dims and strides if user did not set them
+ if (scale->get_dim().empty()) {
+ scale->set_dim(scale_bias_dim);
+ }
+ if (scale->get_stride().empty()) {
+ auto const& scale_dim = scale->get_dim();
+ std::vector<int64_t> stride_order;
+ CHECK_CUDNN_FRONTEND_ERROR(
+ detail::generate_stride_order_preserving_format(X->get_stride(), scale_dim.size(), stride_order));
+ scale->set_stride(detail::generate_stride(scale_dim, stride_order));
+ }
+
+ auto bias = attributes.inputs[Layernorm_attributes::input_names::BIAS];
+ // Only infer dims and strides if user did not set them
+ if (bias->get_dim().empty()) {
+ bias->set_dim(scale_bias_dim);
+ }
+ if (bias->get_stride().empty()) {
+ auto const& bias_dim = bias->get_dim();
+ std::vector<int64_t> stride_order;
+ CHECK_CUDNN_FRONTEND_ERROR(
+ detail::generate_stride_order_preserving_format(X->get_stride(), bias_dim.size(), stride_order));
+ bias->set_stride(detail::generate_stride(bias_dim, stride_order));
+ }
+
+ if (attributes.forward_phase == NormFwdPhase_t::TRAINING) {
+ // stats dim is x where scale == 1 else 1
+ auto stats_dim = X->get_dim();
+ for (size_t i = 0; i < stats_dim.size(); i++) {
+ if (scale->get_dim()[i] != 1) {
+ stats_dim[i] = 1;
+ }
+ }
+
+ auto mean = attributes.outputs[Layernorm_attributes::output_names::MEAN];
+ // Only infer dims and strides if user did not set them
+ if (mean->get_dim().empty()) {
+ mean->set_dim(stats_dim);
+ }
+ if (mean->get_stride().empty()) {
+ auto const& mean_dim = mean->get_dim();
+ std::vector<int64_t> stride_order;
+ CHECK_CUDNN_FRONTEND_ERROR(
+ detail::generate_stride_order_preserving_format(X->get_stride(), mean_dim.size(), stride_order));
+ mean->set_stride(detail::generate_stride(mean_dim, stride_order));
+ }
+
+ auto inv_var = attributes.outputs[Layernorm_attributes::output_names::INV_VARIANCE];
+ // Only infer dims and strides if user did not set them
+ if (inv_var->get_dim().empty()) {
+ inv_var->set_dim(stats_dim);
+ }
+ if (inv_var->get_stride().empty()) {
+ auto const& inv_var_dim = inv_var->get_dim();
+ std::vector<int64_t> stride_order;
+ CHECK_CUDNN_FRONTEND_ERROR(
+ detail::generate_stride_order_preserving_format(X->get_stride(), inv_var_dim.size(), stride_order));
+ inv_var->set_stride(detail::generate_stride(inv_var_dim, stride_order));
+ }
+ }
+
+ // Set scalar tensors
+ std::vector<int64_t> ones(X->get_dim().size(), 1);
+ auto infer_scalar_tensors = [&ones](std::shared_ptr<Tensor_attributes>& T) {
+ // Only infer dims and strides if user did not set them
+ if (T->get_dim().empty()) {
+ T->set_dim(ones);
+ }
+ if (T->get_stride().empty()) {
+ T->set_stride(ones);
+ }
+ };
+ infer_scalar_tensors(attributes.inputs[Layernorm_attributes::input_names::EPSILON]);
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ pre_validate_node() const override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: " << "Validating LayerNormNode " << attributes.name);
+
+ // Norm forward phase should be set
+ RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.forward_phase == NormFwdPhase_t::NOT_SET,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "Forward phase not set of layernorm node.");
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ create_cudnn_operations(
+ std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations,
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations,
+ managed_backend_descriptor_t& raw_operations,
+ std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final {
+ CUDNN_FRONTEND_UNUSED(raw_operations);
+ CUDNN_FE_LOG_LABEL("INFO: " << "Building LayerNormNode operations " << attributes.name << " ");
+
+ // Create operation by directly calling cuDNN backend API
+ Operation_v8 layernorm_operation;
+
+ _CUDNN_CHECK_CUDNN_ERROR(
+ layernorm_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_NORM_FORWARD_DESCRIPTOR));
+
+ // Set norm mode
+ cudnnBackendNormMode_t cudnn_norm_mode;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormMode_t::LAYER_NORM, cudnn_norm_mode));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(layernorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_MODE,
+ CUDNN_TYPE_NORM_MODE,
+ 1,
+ &cudnn_norm_mode));
+
+ // Set forward phase
+ cudnnBackendNormFwdPhase_t cudnn_norm_fwd_phase;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.forward_phase, cudnn_norm_fwd_phase));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(layernorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_PHASE,
+ CUDNN_TYPE_NORM_FWD_PHASE,
+ 1,
+ &cudnn_norm_fwd_phase));
+
+ // Set input tensor X
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Layernorm_attributes::input_names::X);
+ auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(layernorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_XDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &x_desc));
+
+ // Set scale and bias tensors
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SCALE, Layernorm_attributes::input_names::SCALE);
+ auto scale_desc = tensors.at(SCALE->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(layernorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_SCALE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &scale_desc));
+
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(BIAS, Layernorm_attributes::input_names::BIAS);
+ auto bias_desc = tensors.at(BIAS->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(layernorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_BIAS_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &bias_desc));
+
+ // Set epsilon tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(EPSILON, Layernorm_attributes::input_names::EPSILON);
+ auto epsilon_desc = tensors.at(EPSILON->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(layernorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_EPSILON_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &epsilon_desc));
+
+ // Set output tensor Y
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(Y, Layernorm_attributes::output_names::Y);
+ auto y_desc = tensors.at(Y->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(layernorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_YDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &y_desc));
+
+ // Set mean and inv_variance for training phase
+ if (attributes.forward_phase == NormFwdPhase_t::TRAINING) {
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(MEAN, Layernorm_attributes::output_names::MEAN);
+ auto mean_desc = tensors.at(MEAN->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(layernorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_MEAN_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &mean_desc));
+
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(INV_VARIANCE, Layernorm_attributes::output_names::INV_VARIANCE);
+ auto inv_var_desc = tensors.at(INV_VARIANCE->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(layernorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_INV_VARIANCE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &inv_var_desc));
+ }
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(layernorm_operation.get_raw_desc()));
+
+ operations.push_back(std::make_shared<Operation_v8>(std::move(layernorm_operation)));
+
+ auto const& non_virtual_uids = attributes.get_non_virtual_uids();
+ uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end());
+ return {error_code_t::OK, ""};
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ j.update(R"( {"tag": "LAYER_NORM"})"_json);
+ }
+#endif
+};
+
+} // namespace graph
+
+} // namespace cudnn_frontend \ No newline at end of file
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/matmul.h b/deps/cudnn-frontend/include/cudnn_frontend/node/matmul.h
new file mode 100644
index 0000000..f09d341
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/node/matmul.h
@@ -0,0 +1,253 @@
+#pragma once
+
+#include "../graph_helpers.h"
+#include "../node_interface.h"
+
+namespace cudnn_frontend::graph {
+
+class MatmulNode : public NodeCRTP<MatmulNode> {
+ public:
+ Matmul_attributes attributes;
+
+ MatmulNode(Matmul_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP(context), attributes(std::move(attributes_)) {}
+
+ Type
+ getType() override final {
+ return Type::MATMUL;
+ }
+
+ error_t
+ infer_properties_node() override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for matmul node " << attributes.name);
+
+ attributes.fill_from_context(context);
+
+ // Only inferrencing from (A, B) -> C works today.
+ auto a_tensor = attributes.inputs[Matmul_attributes::input_names::A];
+ auto b_tensor = attributes.inputs[Matmul_attributes::input_names::B];
+ auto c_tensor = attributes.outputs[Matmul_attributes::output_names::C];
+
+ auto const a_tensor_dim = a_tensor->get_dim();
+ auto const b_tensor_dim = b_tensor->get_dim();
+ auto c_tensor_dim = c_tensor->get_dim();
+
+ // Only infer dims and strides if user did not set them
+ if (c_tensor_dim.empty()) {
+ // CHECK_CUDNN_FRONTEND_ERROR(detail::generate_matmul_output_dim(a_tensor_dim, b_tensor_dim, c_tensor_dim));
+
+ c_tensor_dim.resize(a_tensor_dim.size());
+ int64_t gemm_start_dim = a_tensor_dim.size() - 2;
+ c_tensor_dim[gemm_start_dim] = a_tensor_dim[gemm_start_dim]; // M
+ c_tensor_dim[gemm_start_dim + 1] = b_tensor_dim[gemm_start_dim + 1]; // N
+
+ // Broadcast the batches
+ for (int64_t i = 0; i < gemm_start_dim; ++i) {
+ c_tensor_dim[i] = std::max(a_tensor_dim[i], b_tensor_dim[i]);
+ }
+
+ c_tensor->set_dim(c_tensor_dim);
+ }
+ if (c_tensor->get_stride().empty()) {
+ auto const& c_dim = c_tensor->get_dim();
+ // Default to Col major
+ auto const& stride_order = detail::generate_row_major_stride_order(c_dim.size());
+ c_tensor->set_stride(detail::generate_stride(c_dim, stride_order));
+ }
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ create_cudnn_operations(
+ std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations,
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations,
+ managed_backend_descriptor_t& raw_operations,
+ std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final {
+ CUDNN_FRONTEND_UNUSED(raw_operations);
+ CUDNN_FE_LOG_LABEL("INFO: " << "Building MatmulNode operations " << attributes.name << " ");
+
+ // Create matmul descriptor by directly calling cuDNN backend API
+ MatMulDesc_v8 matmul_descriptor;
+
+ _CUDNN_CHECK_CUDNN_ERROR(matmul_descriptor.initialize_managed_backend_pointer(CUDNN_BACKEND_MATMUL_DESCRIPTOR));
+
+ // Set compute type
+ cudnnDataType_t cudnn_data_type;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.compute_data_type, cudnn_data_type));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(
+ matmul_descriptor.get_raw_desc(), CUDNN_ATTR_MATMUL_COMP_TYPE, CUDNN_TYPE_DATA_TYPE, 1, &cudnn_data_type));
+
+ // Set padding value if specified
+#if (CUDNN_VERSION >= 8900)
+ if (attributes.padding_value != 0.0) {
+ double padding_value = attributes.padding_value;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(matmul_descriptor.get_raw_desc(),
+ CUDNN_ATTR_MATMUL_PADDING_VALUE,
+ CUDNN_TYPE_DOUBLE,
+ 1,
+ &padding_value));
+ }
+#endif
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(matmul_descriptor.get_raw_desc()));
+ CUDNN_FE_LOG_LABEL_ENDL(matmul_descriptor);
+
+ // Create operation by directly calling cuDNN backend API
+ Operation_v8 matmul_operation;
+
+ _CUDNN_CHECK_CUDNN_ERROR(
+ matmul_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_MATMUL_DESCRIPTOR));
+
+ // Set input tensor A
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(A, Matmul_attributes::input_names::A);
+ auto a_desc = tensors.at(A->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(matmul_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_MATMUL_ADESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &a_desc));
+
+ // Set input tensor B
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(B, Matmul_attributes::input_names::B);
+ auto b_desc = tensors.at(B->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(matmul_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_MATMUL_BDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &b_desc));
+
+ // Set output tensor C
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(C, Matmul_attributes::output_names::C);
+ auto c_desc = tensors.at(C->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(matmul_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_MATMUL_CDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &c_desc));
+
+ // Set matmul descriptor
+ auto matmul_desc_ptr = matmul_descriptor.get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(matmul_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_MATMUL_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &matmul_desc_ptr));
+
+ // Set optional override tensors
+ auto M_override = attributes.inputs.find(Matmul_attributes::input_names::M_override);
+ if ((M_override != attributes.inputs.end()) && (M_override->second != nullptr)) {
+ auto m_override_desc = tensors.at(M_override->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(matmul_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_MATMUL_GEMM_M_OVERRIDE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &m_override_desc));
+ }
+
+ auto N_override = attributes.inputs.find(Matmul_attributes::input_names::N_override);
+ if ((N_override != attributes.inputs.end()) && (N_override->second != nullptr)) {
+ auto n_override_desc = tensors.at(N_override->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(matmul_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_MATMUL_GEMM_N_OVERRIDE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &n_override_desc));
+ }
+
+ auto K_override = attributes.inputs.find(Matmul_attributes::input_names::K_override);
+ if ((K_override != attributes.inputs.end()) && (K_override->second != nullptr)) {
+ auto k_override_desc = tensors.at(K_override->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(matmul_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_MATMUL_GEMM_K_OVERRIDE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &k_override_desc));
+ }
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(matmul_operation.get_raw_desc()));
+
+ operations.push_back(std::make_shared<Operation_v8>(std::move(matmul_operation)));
+
+ auto const& non_virtual_uids = attributes.get_non_virtual_uids();
+ uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end());
+ return {error_code_t::OK, ""};
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ j.update(R"( {"tag": "MATMUL"})"_json);
+ }
+#endif
+};
+
+inline void
+INode::matmul(std::shared_ptr<Tensor_attributes> a,
+ std::shared_ptr<Tensor_attributes> b,
+ Matmul_attributes attributes,
+ std::shared_ptr<Tensor_attributes> c) {
+ attributes.inputs[Matmul_attributes::input_names::A] = a;
+ attributes.inputs[Matmul_attributes::input_names::B] = b;
+ attributes.outputs[Matmul_attributes::output_names::C] = c;
+ sub_nodes.emplace_back(std::make_unique<MatmulNode>(std::move(attributes), context));
+}
+
+inline std::shared_ptr<Tensor_attributes>
+INode::matmul(std::shared_ptr<Tensor_attributes> a,
+ std::shared_ptr<Tensor_attributes> b,
+ Matmul_attributes attributes) {
+ if (attributes.name.empty()) {
+ attributes.name += std::to_string(sub_nodes.size());
+ }
+ attributes.inputs[Matmul_attributes::input_names::A] = a;
+ attributes.inputs[Matmul_attributes::input_names::B] = b;
+
+ if (a->get_name().empty()) {
+ a->set_name(attributes.name + "::A");
+ };
+ if (b->get_name().empty()) {
+ b->set_name(attributes.name + "::B");
+ };
+
+ auto m_override = attributes.inputs.find(Matmul_attributes::input_names::M_override);
+ auto n_override = attributes.inputs.find(Matmul_attributes::input_names::N_override);
+ auto k_override = attributes.inputs.find(Matmul_attributes::input_names::K_override);
+
+ if (m_override != attributes.inputs.end()) {
+ auto tensor = m_override->second;
+ if (tensor && tensor->get_name().empty()) {
+ tensor->set_name(attributes.name + "::M_override");
+ }
+ }
+ if (n_override != attributes.inputs.end()) {
+ auto tensor = n_override->second;
+ if (tensor && tensor->get_name().empty()) {
+ tensor->set_name(attributes.name + "::N_override");
+ }
+ }
+ if (k_override != attributes.inputs.end()) {
+ auto tensor = k_override->second;
+ if (tensor && tensor->get_name().empty()) {
+ tensor->set_name(attributes.name + "::K_override");
+ }
+ }
+
+ auto C = attributes.outputs[Matmul_attributes::output_names::C] = output_tensor(attributes.name + "::C");
+
+ sub_nodes.emplace_back(std::make_unique<MatmulNode>(std::move(attributes), context));
+ return C;
+}
+
+} // namespace cudnn_frontend::graph \ No newline at end of file
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/matmul_fp8.h b/deps/cudnn-frontend/include/cudnn_frontend/node/matmul_fp8.h
new file mode 100644
index 0000000..d3fe5e5
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/node/matmul_fp8.h
@@ -0,0 +1,104 @@
+#pragma once
+
+#include "../graph_helpers.h"
+#include "../node_interface.h"
+
+namespace cudnn_frontend::graph {
+
+class MatmulFP8Node : public NodeCRTP<MatmulFP8Node> {
+ public:
+ Matmul_fp8_attributes attributes;
+
+ MatmulFP8Node(Matmul_fp8_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP(context), attributes(std::move(attributes_)) {}
+
+ Type
+ getType() override final {
+ return Type::MATMUL;
+ }
+
+ error_t
+ infer_properties_node() override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for matmul fp8 node " << attributes.name);
+
+ attributes.fill_from_context(context);
+
+ auto const& a_dim = attributes.inputs.at(Matmul_fp8_attributes::input_names::A)->get_dim();
+ auto const& b_dim = attributes.inputs.at(Matmul_fp8_attributes::input_names::B)->get_dim();
+ auto const& c_dim = attributes.outputs.at(Matmul_fp8_attributes::output_names::C)->get_dim();
+
+ std::shared_ptr<Tensor_attributes> last_output;
+
+ // Matmul
+
+ auto matmul_attributes = Matmul_attributes();
+ matmul_attributes.clone_fp8_attributes(attributes);
+ matmul_attributes.set_name("matmul");
+
+ last_output = matmul(attributes.inputs.at(Matmul_fp8_attributes::input_names::A),
+ attributes.inputs.at(Matmul_fp8_attributes::input_names::B),
+ matmul_attributes);
+
+ // Reduction if GQA for head dimension
+ if (a_dim.size() == 4 && b_dim.size() == 4 && c_dim.size() == 4 && a_dim[1] == b_dim[1] &&
+ a_dim[1] != c_dim[1] && (a_dim[1] % c_dim[1] == 0)) {
+ auto gqa_attributes = Reduction_attributes().set_name("gqa_c").set_mode(ReductionMode_t::ADD);
+ last_output = reduction(last_output, gqa_attributes);
+ last_output->set_dim(c_dim);
+ }
+
+ //// Scale Descales
+ auto mul_attributes = Pointwise_attributes().set_mode(PointwiseMode_t::MUL);
+ // Descale A
+ mul_attributes.set_name("descale_a");
+ last_output =
+ pointwise(last_output, attributes.inputs.at(Matmul_fp8_attributes::input_names::Descale_A), mul_attributes);
+
+ // Descale B
+ mul_attributes.set_name("descale_b");
+ last_output =
+ pointwise(last_output, attributes.inputs.at(Matmul_fp8_attributes::input_names::Descale_B), mul_attributes);
+
+ // Scale C
+ mul_attributes.set_name("scale_c");
+ // Special non-functional-style call. Needed because output already created and provided to user.
+ pointwise(last_output,
+ attributes.inputs.at(Matmul_fp8_attributes::input_names::Scale_C),
+ mul_attributes,
+ attributes.outputs.at(Matmul_fp8_attributes::output_names::C));
+
+ // Amax C
+ auto amax_attributes = Reduction_attributes().set_name("amax_c").set_mode(ReductionMode_t::AMAX);
+ // Special non-functional-style call. Needed because output already created and provided to user.
+ reduction(last_output, amax_attributes, attributes.outputs.at(Matmul_fp8_attributes::output_names::Amax_C));
+
+ return {error_code_t::OK, ""};
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ j.update(R"( {"tag": "MATMUL_FP8"})"_json);
+ }
+#endif
+};
+inline void
+INode::matmul_fp8(std::shared_ptr<Tensor_attributes> a,
+ std::shared_ptr<Tensor_attributes> b,
+ std::shared_ptr<Tensor_attributes> descale_a,
+ std::shared_ptr<Tensor_attributes> descale_b,
+ std::shared_ptr<Tensor_attributes> scale_c,
+ Matmul_fp8_attributes attributes,
+ std::shared_ptr<Tensor_attributes> c,
+ std::shared_ptr<Tensor_attributes> amax_c) {
+ attributes.inputs[Matmul_fp8_attributes::input_names::A] = a;
+ attributes.inputs[Matmul_fp8_attributes::input_names::B] = b;
+ attributes.inputs[Matmul_fp8_attributes::input_names::Descale_A] = descale_a;
+ attributes.inputs[Matmul_fp8_attributes::input_names::Descale_B] = descale_b;
+ attributes.inputs[Matmul_fp8_attributes::input_names::Scale_C] = scale_c;
+ attributes.outputs[Matmul_fp8_attributes::output_names::C] = c;
+ attributes.outputs[Matmul_fp8_attributes::output_names::Amax_C] = amax_c;
+ sub_nodes.emplace_back(std::make_unique<MatmulFP8Node>(std::move(attributes), context));
+}
+} // namespace cudnn_frontend::graph \ No newline at end of file
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/moe_grouped_matmul.h b/deps/cudnn-frontend/include/cudnn_frontend/node/moe_grouped_matmul.h
new file mode 100644
index 0000000..d05d75d
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/node/moe_grouped_matmul.h
@@ -0,0 +1,221 @@
+#pragma once
+
+#include "../graph_helpers.h"
+#include "../node_interface.h"
+
+namespace cudnn_frontend::graph {
+
+class MoeGroupedMatmulNode : public NodeCRTP<MoeGroupedMatmulNode> {
+ public:
+ Moe_grouped_matmul_attributes attributes;
+
+ MoeGroupedMatmulNode(Moe_grouped_matmul_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP(context), attributes(std::move(attributes_)) {}
+
+ Type
+ getType() override final {
+ return Type::MOE_GROUPED_MATMUL;
+ }
+
+ error_t
+ pre_validate_node() const override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating MoeGroupedMatmulNode " << attributes.name);
+
+ auto const token_it = attributes.inputs.find(Moe_grouped_matmul_attributes::input_names::Token);
+ RETURN_CUDNN_FRONTEND_ERROR_IF(token_it == attributes.inputs.end() || token_it->second == nullptr,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MoeGroupedMatmul input Token not set.");
+
+ auto const weight_it = attributes.inputs.find(Moe_grouped_matmul_attributes::input_names::Weight);
+ RETURN_CUDNN_FRONTEND_ERROR_IF(weight_it == attributes.inputs.end() || weight_it->second == nullptr,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MoeGroupedMatmul input Weight not set.");
+
+ auto const first_token_offset_it =
+ attributes.inputs.find(Moe_grouped_matmul_attributes::input_names::FirstTokenOffset);
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ first_token_offset_it == attributes.inputs.end() || first_token_offset_it->second == nullptr,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MoeGroupedMatmul input FirstTokenOffset not set.");
+
+ auto const output_it = attributes.outputs.find(Moe_grouped_matmul_attributes::output_names::Output);
+ RETURN_CUDNN_FRONTEND_ERROR_IF(output_it == attributes.outputs.end() || output_it->second == nullptr,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MoeGroupedMatmul output Output not set.");
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ infer_properties_node() override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for moe grouped matmul node " << attributes.name);
+
+ attributes.fill_from_context(context);
+
+ auto token_tensor = attributes.inputs[Moe_grouped_matmul_attributes::input_names::Token];
+ auto weight_tensor = attributes.inputs[Moe_grouped_matmul_attributes::input_names::Weight];
+ auto token_index_tensor = attributes.inputs[Moe_grouped_matmul_attributes::input_names::TokenIndex];
+ auto output_tensor = attributes.outputs[Moe_grouped_matmul_attributes::output_names::Output];
+
+ auto const token_tensor_dim = token_tensor->get_dim();
+ auto const weight_tensor_dim = weight_tensor->get_dim();
+ auto output_tensor_dim = output_tensor->get_dim();
+
+ if (output_tensor_dim.empty()) {
+ output_tensor_dim.resize(3);
+ output_tensor_dim[0] = 1;
+ output_tensor_dim[2] = weight_tensor_dim[2];
+ if (attributes.mode == MoeGroupedMatmulMode_t::GATHER) {
+ output_tensor_dim[1] = token_index_tensor->get_dim()[1];
+ } else {
+ output_tensor_dim[1] = token_tensor_dim[1];
+ }
+ output_tensor_dim.resize(3);
+
+ output_tensor->set_dim(output_tensor_dim);
+ }
+
+ if (output_tensor->get_stride().empty()) {
+ auto const& output_dim = output_tensor->get_dim();
+ auto const& stride_order = detail::generate_row_major_stride_order(output_dim.size());
+ output_tensor->set_stride(detail::generate_stride(output_dim, stride_order));
+ }
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ create_cudnn_operations(
+ std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations,
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations,
+ managed_backend_descriptor_t& raw_operations,
+ std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final {
+ getLogger() << "[cudnn_frontend] INFO: "
+ << "Building MoeGroupedMatmulNode operations " << attributes.name << std::endl;
+ auto cudnn_ver_error = error_t{error_code_t::GRAPH_NOT_SUPPORTED, "Moe grouped matmul requires cuDNN v9.15.0"};
+
+#if (CUDNN_VERSION >= 91500)
+ NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(91500, cudnn_ver_error);
+ CUDNN_FRONTEND_UNUSED(operations);
+
+ auto moe_grouped_matmul_operation =
+ make_shared_backend_pointer(CUDNN_BACKEND_OPERATION_MOE_GROUPED_MATMUL_DESCRIPTOR);
+
+ cudnnMoeGroupedMatmulMode_t moe_grouped_matmul_mode;
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.mode, moe_grouped_matmul_mode));
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_MODE,
+ CUDNN_TYPE_MOE_GROUPED_MATMUL_MODE,
+ 1,
+ &moe_grouped_matmul_mode));
+
+ cudnnDataType_t cudnn_data_type;
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.compute_data_type, cudnn_data_type));
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_MATH_PREC,
+ CUDNN_TYPE_DATA_TYPE,
+ 1,
+ &cudnn_data_type));
+
+ auto token = attributes.inputs.find(Moe_grouped_matmul_attributes::input_names::Token)->second;
+ auto backend_token = tensors[token->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_TOKEN_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_token));
+
+ auto weight = attributes.inputs.find(Moe_grouped_matmul_attributes::input_names::Weight)->second;
+ auto backend_weight = tensors[weight->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_WEIGHT_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_weight));
+
+ auto first_token_offset =
+ attributes.inputs.find(Moe_grouped_matmul_attributes::input_names::FirstTokenOffset)->second;
+ auto backend_first_token_offset = tensors[first_token_offset->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_FIRST_TOKEN_OFFSET_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_first_token_offset));
+
+ auto output = attributes.outputs.find(Moe_grouped_matmul_attributes::output_names::Output)->second;
+ auto backend_output = tensors[output->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_OUTPUT_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_output));
+
+ if (attributes.mode == MoeGroupedMatmulMode_t::GATHER || attributes.mode == MoeGroupedMatmulMode_t::SCATTER) {
+ auto token_index = attributes.inputs.find(Moe_grouped_matmul_attributes::input_names::TokenIndex)->second;
+ auto backend_token_index = tensors[token_index->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_TOKEN_INDEX_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_token_index));
+ }
+
+ if (attributes.mode == MoeGroupedMatmulMode_t::SCATTER) {
+ auto token_ks = attributes.inputs.find(Moe_grouped_matmul_attributes::input_names::TokenKs)->second;
+ auto backend_token_ks = tensors[token_ks->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_TOKEN_KS_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_token_ks));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_TOP_K,
+ CUDNN_TYPE_INT32,
+ 1,
+ &(attributes.top_k)));
+ }
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(moe_grouped_matmul_operation->get_backend_descriptor()));
+
+ raw_operations.push_back(moe_grouped_matmul_operation);
+
+ auto const& non_virtual_uids = attributes.get_non_virtual_uids();
+ uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end());
+ return {error_code_t::OK, ""};
+#else
+ CUDNN_FRONTEND_UNUSED(uids_involved_in_operations);
+ CUDNN_FRONTEND_UNUSED(operations);
+ CUDNN_FRONTEND_UNUSED(raw_operations);
+ CUDNN_FRONTEND_UNUSED(tensors);
+ return cudnn_ver_error;
+#endif
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ j.update(R"( {"tag": "MOE_GROUPED_MATMUL"})"_json);
+ }
+#endif
+};
+
+inline void
+INode::moe_grouped_matmul(std::shared_ptr<Tensor_attributes> token,
+ std::shared_ptr<Tensor_attributes> weight,
+ std::shared_ptr<Tensor_attributes> first_token_offset,
+ std::shared_ptr<Tensor_attributes> token_index,
+ std::shared_ptr<Tensor_attributes> token_ks,
+ Moe_grouped_matmul_attributes attributes,
+ std::shared_ptr<Tensor_attributes> output) {
+ attributes.inputs[Moe_grouped_matmul_attributes::input_names::Token] = token;
+ attributes.inputs[Moe_grouped_matmul_attributes::input_names::Weight] = weight;
+ attributes.inputs[Moe_grouped_matmul_attributes::input_names::FirstTokenOffset] = first_token_offset;
+ attributes.inputs[Moe_grouped_matmul_attributes::input_names::TokenIndex] = token_index;
+ attributes.inputs[Moe_grouped_matmul_attributes::input_names::TokenKs] = token_ks;
+ attributes.outputs[Moe_grouped_matmul_attributes::output_names::Output] = output;
+ sub_nodes.emplace_back(std::make_unique<MoeGroupedMatmulNode>(std::move(attributes), context));
+}
+
+} // namespace cudnn_frontend::graph \ No newline at end of file
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/moe_grouped_matmul_bwd.h b/deps/cudnn-frontend/include/cudnn_frontend/node/moe_grouped_matmul_bwd.h
new file mode 100644
index 0000000..2076203
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/node/moe_grouped_matmul_bwd.h
@@ -0,0 +1,181 @@
+#pragma once
+
+#include "../graph_helpers.h"
+#include "../node_interface.h"
+
+namespace cudnn_frontend::graph {
+
+class MoeGroupedMatmulBwdNode : public NodeCRTP<MoeGroupedMatmulBwdNode> {
+ public:
+ Moe_grouped_matmul_bwd_attributes attributes;
+
+ MoeGroupedMatmulBwdNode(Moe_grouped_matmul_bwd_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP(context), attributes(std::move(attributes_)) {}
+
+ Type
+ getType() override final {
+ return Type::MOE_GROUPED_MATMUL_BWD;
+ }
+
+ error_t
+ pre_validate_node() const override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating MoeGroupedMatmulBwdNode " << attributes.name);
+
+ auto const doutput_it = attributes.inputs.find(Moe_grouped_matmul_bwd_attributes::input_names::DOutput);
+ RETURN_CUDNN_FRONTEND_ERROR_IF(doutput_it == attributes.inputs.end() || doutput_it->second == nullptr,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MoeGroupedMatmulBwd input DOutput not set.");
+
+ auto const token_it = attributes.inputs.find(Moe_grouped_matmul_bwd_attributes::input_names::Token);
+ RETURN_CUDNN_FRONTEND_ERROR_IF(token_it == attributes.inputs.end() || token_it->second == nullptr,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MoeGroupedMatmulBwd input Token not set.");
+
+ auto const first_token_offset_it =
+ attributes.inputs.find(Moe_grouped_matmul_bwd_attributes::input_names::FirstTokenOffset);
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ first_token_offset_it == attributes.inputs.end() || first_token_offset_it->second == nullptr,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MoeGroupedMatmulBwd input FirstTokenOffset not set.");
+
+ auto const dweight_it = attributes.outputs.find(Moe_grouped_matmul_bwd_attributes::output_names::DWeight);
+ RETURN_CUDNN_FRONTEND_ERROR_IF(dweight_it == attributes.outputs.end() || dweight_it->second == nullptr,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MoeGroupedMatmulBwd output DWeight not set.");
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ infer_properties_node() override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for moe grouped matmul bwd node "
+ << attributes.name);
+
+ attributes.fill_from_context(context);
+
+ auto doutput_tensor = attributes.inputs[Moe_grouped_matmul_bwd_attributes::input_names::DOutput];
+ auto token_tensor = attributes.inputs[Moe_grouped_matmul_bwd_attributes::input_names::Token];
+ auto token_index_tensor = attributes.inputs[Moe_grouped_matmul_bwd_attributes::input_names::FirstTokenOffset];
+ auto dweight_tensor = attributes.outputs[Moe_grouped_matmul_bwd_attributes::output_names::DWeight];
+
+ auto const doutput_tensor_dim = doutput_tensor->get_dim();
+ auto const token_tensor_dim = token_tensor->get_dim();
+ auto const token_index_tensor_dim = token_index_tensor->get_dim();
+ auto dweight_tensor_dim = dweight_tensor->get_dim();
+
+ if (dweight_tensor_dim.empty()) {
+ dweight_tensor_dim.resize(3);
+ dweight_tensor_dim[0] = token_index_tensor_dim[0];
+ dweight_tensor_dim[1] = token_tensor_dim[2];
+ dweight_tensor_dim[2] = doutput_tensor_dim[2];
+ dweight_tensor->set_dim(dweight_tensor_dim);
+ }
+
+ if (dweight_tensor->get_stride().empty()) {
+ auto const& dweight_dim = dweight_tensor->get_dim();
+ auto const& stride_order = detail::generate_column_major_stride_order(dweight_dim.size());
+ dweight_tensor->set_stride(detail::generate_stride(dweight_dim, stride_order));
+ }
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ create_cudnn_operations(
+ std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations,
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations,
+ managed_backend_descriptor_t& raw_operations,
+ std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final {
+ getLogger() << "[cudnn_frontend] INFO: "
+ << "Building MoeGroupedMatmulBwdNode operations " << attributes.name << std::endl;
+ auto cudnn_ver_error =
+ error_t{error_code_t::GRAPH_NOT_SUPPORTED, "Moe grouped matmul bwd requires cuDNN v9.22.0"};
+
+#if (CUDNN_VERSION >= 92200)
+ NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(92200, cudnn_ver_error);
+ CUDNN_FRONTEND_UNUSED(operations);
+
+ auto moe_grouped_matmul_bwd_operation =
+ make_shared_backend_pointer(CUDNN_BACKEND_OPERATION_MOE_GROUPED_MATMUL_BWD_DESCRIPTOR);
+
+ cudnnDataType_t cudnn_data_type;
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.compute_data_type, cudnn_data_type));
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_bwd_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_BWD_MATH_PREC,
+ CUDNN_TYPE_DATA_TYPE,
+ 1,
+ &cudnn_data_type));
+
+ auto token = attributes.inputs.find(Moe_grouped_matmul_bwd_attributes::input_names::Token)->second;
+ auto backend_token = tensors[token->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_bwd_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_BWD_TOKEN_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_token));
+
+ auto dweight = attributes.outputs.find(Moe_grouped_matmul_bwd_attributes::output_names::DWeight)->second;
+ auto backend_dweight = tensors[dweight->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_bwd_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_BWD_DWEIGHT_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_dweight));
+
+ auto first_token_offset =
+ attributes.inputs.find(Moe_grouped_matmul_bwd_attributes::input_names::FirstTokenOffset)->second;
+ auto backend_first_token_offset = tensors[first_token_offset->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(
+ detail::set_attribute(moe_grouped_matmul_bwd_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_BWD_FIRST_TOKEN_OFFSET_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_first_token_offset));
+
+ auto doutput = attributes.inputs.find(Moe_grouped_matmul_bwd_attributes::input_names::DOutput)->second;
+ auto backend_doutput = tensors[doutput->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_bwd_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_BWD_DOUTPUT_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_doutput));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(moe_grouped_matmul_bwd_operation->get_backend_descriptor()));
+
+ raw_operations.push_back(moe_grouped_matmul_bwd_operation);
+
+ auto const& non_virtual_uids = attributes.get_non_virtual_uids();
+ uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end());
+ return {error_code_t::OK, ""};
+#else
+ CUDNN_FRONTEND_UNUSED(uids_involved_in_operations);
+ CUDNN_FRONTEND_UNUSED(operations);
+ CUDNN_FRONTEND_UNUSED(raw_operations);
+ CUDNN_FRONTEND_UNUSED(tensors);
+ return cudnn_ver_error;
+#endif
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ j.update(R"( {"tag": "MOE_GROUPED_MATMUL_BWD"})"_json);
+ }
+#endif
+};
+
+inline void
+INode::moe_grouped_matmul_bwd(std::shared_ptr<Tensor_attributes> doutput,
+ std::shared_ptr<Tensor_attributes> token,
+ std::shared_ptr<Tensor_attributes> first_token_offset,
+ Moe_grouped_matmul_bwd_attributes attributes,
+ std::shared_ptr<Tensor_attributes> dweight) {
+ attributes.inputs[Moe_grouped_matmul_bwd_attributes::input_names::DOutput] = doutput;
+ attributes.inputs[Moe_grouped_matmul_bwd_attributes::input_names::Token] = token;
+ attributes.inputs[Moe_grouped_matmul_bwd_attributes::input_names::FirstTokenOffset] = first_token_offset;
+ attributes.outputs[Moe_grouped_matmul_bwd_attributes::output_names::DWeight] = dweight;
+ sub_nodes.emplace_back(std::make_unique<MoeGroupedMatmulBwdNode>(std::move(attributes), context));
+}
+
+} // namespace cudnn_frontend::graph \ No newline at end of file
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/paged_cache_load.h b/deps/cudnn-frontend/include/cudnn_frontend/node/paged_cache_load.h
new file mode 100644
index 0000000..9fc8610
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/node/paged_cache_load.h
@@ -0,0 +1,153 @@
+#pragma once
+
+#include "../graph_helpers.h"
+#include "../node_interface.h"
+
+#include "pointwise.h"
+#include "reduction.h"
+
+namespace cudnn_frontend::graph {
+
+class PagedCacheLoadNode : public NodeCRTP<PagedCacheLoadNode> {
+ public:
+ PagedCacheLoad_attributes attributes;
+
+ PagedCacheLoadNode(PagedCacheLoad_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP(context), attributes(std::move(attributes_)) {}
+
+ Type
+ getType() override final {
+ return Type::PAGED_CACHE_LOAD;
+ }
+
+ error_t
+ create_cudnn_operations(
+ std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations,
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations,
+ managed_backend_descriptor_t& raw_operations,
+ std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final {
+ CUDNN_FRONTEND_UNUSED(raw_operations);
+ CUDNN_FE_LOG_LABEL("INFO: " << "Building PagedCacheLoadNode operations " << attributes.name << " ");
+ auto cudnn_ver_error = error_t{error_code_t::GRAPH_NOT_SUPPORTED, "Paged cache load requires cuDNN v9.5.0"};
+
+#if (CUDNN_VERSION >= 90500)
+ NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(90500, cudnn_ver_error);
+
+ // Create operation by directly calling cuDNN backend API
+ Operation_v8 paged_cache_load_operation;
+
+ _CUDNN_CHECK_CUDNN_ERROR(paged_cache_load_operation.initialize_managed_backend_pointer(
+ CUDNN_BACKEND_OPERATION_PAGED_CACHE_LOAD_DESCRIPTOR));
+
+ // Set container tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(container, PagedCacheLoad_attributes::input_names::container);
+ auto container_desc = tensors.at(container->second->get_uid())->get_raw_desc();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(paged_cache_load_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_PAGED_CACHE_LOAD_CONTAINER_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &container_desc));
+
+ // Set page table tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(pageTable, PagedCacheLoad_attributes::input_names::pageTable);
+ auto page_table_desc = tensors.at(pageTable->second->get_uid())->get_raw_desc();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(paged_cache_load_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_PAGED_CACHE_LOAD_PAGE_TABLE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &page_table_desc));
+
+ // Set sequence length tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(seqLen, PagedCacheLoad_attributes::input_names::seqLen);
+ auto seq_len_desc = tensors.at(seqLen->second->get_uid())->get_raw_desc();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(paged_cache_load_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_PAGED_CACHE_LOAD_SEQUENCE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &seq_len_desc));
+
+ // Set output tensor Y
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(yOut, PagedCacheLoad_attributes::output_names::yOut);
+ auto y_desc = tensors.at(yOut->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(paged_cache_load_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_PAGED_CACHE_LOAD_YDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &y_desc));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(paged_cache_load_operation.get_raw_desc()));
+
+ operations.push_back(std::make_shared<Operation_v8>(std::move(paged_cache_load_operation)));
+
+ auto const& non_virtual_uids = attributes.get_non_virtual_uids();
+ uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end());
+
+ return {error_code_t::OK, ""};
+#else
+ CUDNN_FRONTEND_UNUSED(uids_involved_in_operations);
+ CUDNN_FRONTEND_UNUSED(operations);
+ CUDNN_FRONTEND_UNUSED(tensors);
+ return cudnn_ver_error;
+#endif
+ }
+
+ error_t
+ pre_validate_node() const override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating PagedCacheLoadNode " << attributes.name);
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 90500 || detail::get_compiled_version() < 90500,
+ error_code_t::CUDNN_BACKEND_API_FAILED,
+ "The cuDNN backend version must be at least 9.5.0 at compile time and runtime "
+ "in order to use PagedCacheLoadNode.");
+
+ auto const yOut_dims = attributes.outputs.at(PagedCacheLoad_attributes::output_names::yOut)->get_dim();
+ auto const yOut_strides = attributes.outputs.at(PagedCacheLoad_attributes::output_names::yOut)->get_stride();
+ auto const container_dims = attributes.inputs.at(PagedCacheLoad_attributes::input_names::container)->get_dim();
+ auto const blockTable_dims = attributes.inputs.at(PagedCacheLoad_attributes::input_names::pageTable)->get_dim();
+
+ // In the backend, the k-cache is passed as K^T and has dims [B,H,D,S], while v-cache has dims [B,H,S,D]
+ // Use the strides to distinguish.
+ auto yIsTransposed = yOut_strides[2] == 1;
+ auto s_kv = !yIsTransposed ? yOut_dims[2] : yOut_dims[3];
+
+ auto block_size = container_dims[2];
+ auto block_table_size = blockTable_dims[2];
+ bool is_block_table_packed =
+ attributes.inputs.at(PagedCacheLoad_attributes::input_names::pageTable)->get_ragged_offset() != nullptr;
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ !is_block_table_packed && (s_kv + (block_size - 1)) / block_size != block_table_size,
+ error_code_t::INVALID_VALUE,
+ "Paged cache load: block table size must equal ceil(s_kv/block_size), except when using packed block "
+ "tables");
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ infer_properties_node() override final {
+ return {error_code_t::OK, ""};
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ }
+#endif
+};
+
+inline void
+INode::paged_cache_load(std::shared_ptr<Tensor_attributes> container,
+ std::shared_ptr<Tensor_attributes> seqLen,
+ std::shared_ptr<Tensor_attributes> pageTable,
+ PagedCacheLoad_attributes attributes,
+ std::shared_ptr<Tensor_attributes> yOut) {
+ attributes.inputs[PagedCacheLoad_attributes::input_names::container] = std::move(container);
+ attributes.inputs[PagedCacheLoad_attributes::input_names::seqLen] = std::move(seqLen);
+ attributes.inputs[PagedCacheLoad_attributes::input_names::pageTable] = std::move(pageTable);
+ attributes.outputs[PagedCacheLoad_attributes::output_names::yOut] = std::move(yOut);
+ sub_nodes.emplace_back(std::make_unique<PagedCacheLoadNode>(std::move(attributes), context));
+}
+} // namespace cudnn_frontend::graph \ No newline at end of file
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/pointwise.h b/deps/cudnn-frontend/include/cudnn_frontend/node/pointwise.h
new file mode 100644
index 0000000..6d2da14
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/node/pointwise.h
@@ -0,0 +1,390 @@
+#pragma once
+
+#include "../graph_helpers.h"
+#include "../node_interface.h"
+
+namespace cudnn_frontend::graph {
+
+class PointwiseNode : public NodeCRTP<PointwiseNode> {
+ public:
+ Pointwise_attributes attributes;
+
+ PointwiseNode(Pointwise_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP(context), attributes(std::move(attributes_)) {}
+
+ Type
+ getType() override final {
+ return Type::POINTWISE;
+ }
+
+ error_t
+ infer_properties_node() override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for pointwise node " << attributes.name);
+
+ attributes.fill_from_context(context);
+
+ auto out_0_tensor = attributes.outputs.at(Pointwise_attributes::output_names::OUT_0);
+
+ auto output_dim = out_0_tensor->get_dim();
+ // Only infer dims and strides if user did not set them
+ if (output_dim.empty()) {
+ std::vector<std::vector<int64_t>> input_shapes;
+ for (const auto& [input_name, input_tensor] : attributes.inputs) {
+ if (!input_tensor) {
+ continue;
+ }
+ input_shapes.push_back(input_tensor->get_dim());
+ }
+
+ CHECK_CUDNN_FRONTEND_ERROR(detail::compute_broadcast_shape(input_shapes, output_dim));
+ out_0_tensor->set_dim(output_dim);
+ }
+
+ if (out_0_tensor->get_stride().empty()) {
+ for (const auto& [input_name, input_tensor] : attributes.inputs) {
+ if (input_tensor == nullptr) {
+ continue;
+ }
+ if (input_tensor->get_dim() == out_0_tensor->get_dim()) {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO:" << " " << out_0_tensor->get_name()
+ << " stride computed from " << input_tensor->get_name());
+ out_0_tensor->set_stride(input_tensor->get_stride());
+ break;
+ }
+ }
+ if (out_0_tensor->get_stride().empty() && out_0_tensor->get_is_virtual()) {
+ // If the tensor is virtual the strides are immaterial
+ auto input_stride = attributes.inputs.at(Pointwise_attributes::input_names::IN_0)->get_stride();
+ std::vector<int64_t> stride_order;
+ CHECK_CUDNN_FRONTEND_ERROR(
+ detail::generate_stride_order_preserving_format(input_stride, output_dim.size(), stride_order));
+ out_0_tensor->set_stride(detail::generate_stride(output_dim, stride_order));
+ }
+ RETURN_CUDNN_FRONTEND_ERROR_IF(out_0_tensor->get_stride().empty(),
+ error_code_t::SHAPE_DEDUCTION_FAILED,
+ "Pointwise output strides could not be computed");
+ }
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ create_cudnn_operations(
+ std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations,
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations,
+ managed_backend_descriptor_t& raw_operations,
+ std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final {
+ CUDNN_FRONTEND_UNUSED(raw_operations);
+ CUDNN_FE_LOG_LABEL("INFO: " << "Building PointwiseNode operations " << attributes.name << " ");
+
+ // Create pointwise descriptor by directly calling cuDNN backend API
+ PointWiseDesc_v8 pointwise_descriptor;
+
+ _CUDNN_CHECK_CUDNN_ERROR(
+ pointwise_descriptor.initialize_managed_backend_pointer(CUDNN_BACKEND_POINTWISE_DESCRIPTOR));
+
+ // Set pointwise mode
+ cudnnPointwiseMode_t cudnn_pointwise_mode;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.mode, cudnn_pointwise_mode));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_descriptor.get_raw_desc(),
+ CUDNN_ATTR_POINTWISE_MODE,
+ CUDNN_TYPE_POINTWISE_MODE,
+ 1,
+ &cudnn_pointwise_mode));
+
+ // Set compute type
+ cudnnDataType_t cudnn_data_type;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.compute_data_type, cudnn_data_type));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_descriptor.get_raw_desc(),
+ CUDNN_ATTR_POINTWISE_MATH_PREC,
+ CUDNN_TYPE_DATA_TYPE,
+ 1,
+ &cudnn_data_type));
+
+ // Set mode-specific attributes
+ if (attributes.mode == PointwiseMode_t::RELU_FWD || attributes.mode == PointwiseMode_t::RELU_BWD) {
+ cudnnNanPropagation_t nan_propagation = CUDNN_PROPAGATE_NAN;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_descriptor.get_raw_desc(),
+ CUDNN_ATTR_POINTWISE_NAN_PROPAGATION,
+ CUDNN_TYPE_NAN_PROPOGATION,
+ 1,
+ &nan_propagation));
+
+ double lower_clip = attributes.relu_lower_clip.value_or(0.0);
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_descriptor.get_raw_desc(),
+ CUDNN_ATTR_POINTWISE_RELU_LOWER_CLIP,
+ CUDNN_TYPE_DOUBLE,
+ 1,
+ &lower_clip));
+
+ double upper_clip = attributes.relu_upper_clip.value_or(std::numeric_limits<double>::max());
+ if (attributes.compute_data_type == DataType_t::FLOAT) {
+ upper_clip = std::min<double>(upper_clip, std::numeric_limits<float>::max());
+ }
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_descriptor.get_raw_desc(),
+ CUDNN_ATTR_POINTWISE_RELU_UPPER_CLIP,
+ CUDNN_TYPE_DOUBLE,
+ 1,
+ &upper_clip));
+
+ double lower_clip_slope = attributes.relu_lower_clip_slope.value_or(0.0);
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_descriptor.get_raw_desc(),
+ CUDNN_ATTR_POINTWISE_RELU_LOWER_CLIP_SLOPE,
+ CUDNN_TYPE_DOUBLE,
+ 1,
+ &lower_clip_slope));
+ } else if (attributes.mode == PointwiseMode_t::ELU_FWD || attributes.mode == PointwiseMode_t::ELU_BWD) {
+ double elu_alpha = attributes.elu_alpha.value_or(1.0);
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(
+ pointwise_descriptor.get_raw_desc(), CUDNN_ATTR_POINTWISE_ELU_ALPHA, CUDNN_TYPE_DOUBLE, 1, &elu_alpha));
+ } else if (attributes.mode == PointwiseMode_t::SOFTPLUS_FWD ||
+ attributes.mode == PointwiseMode_t::SOFTPLUS_BWD) {
+ double softplus_beta = attributes.softplus_beta.value_or(1.0);
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_descriptor.get_raw_desc(),
+ CUDNN_ATTR_POINTWISE_SOFTPLUS_BETA,
+ CUDNN_TYPE_DOUBLE,
+ 1,
+ &softplus_beta));
+ } else if (attributes.mode == PointwiseMode_t::SWISH_FWD || attributes.mode == PointwiseMode_t::SWISH_BWD) {
+ double swish_beta = attributes.swish_beta.value_or(1.0);
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_descriptor.get_raw_desc(),
+ CUDNN_ATTR_POINTWISE_SWISH_BETA,
+ CUDNN_TYPE_DOUBLE,
+ 1,
+ &swish_beta));
+ } else if (attributes.mode == PointwiseMode_t::GEN_INDEX) {
+ int64_t axis = attributes.get_axis().value_or(-1);
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(
+ pointwise_descriptor.get_raw_desc(), CUDNN_ATTR_POINTWISE_AXIS, CUDNN_TYPE_INT64, 1, &axis));
+ }
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(pointwise_descriptor.get_raw_desc()));
+ CUDNN_FE_LOG_LABEL_ENDL(pointwise_descriptor);
+
+ // Create operation by directly calling cuDNN backend API
+ Operation_v8 pointwise_operation;
+
+ _CUDNN_CHECK_CUDNN_ERROR(
+ pointwise_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR));
+
+ // Set the pointwise descriptor
+ auto pw_desc_ptr = pointwise_descriptor.get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_POINTWISE_PW_DESCRIPTOR,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &pw_desc_ptr));
+
+ auto const port_count = get_pointwise_mode_port_count(attributes.mode);
+ bool const is_activation_bwd = detail::is_activation_backward_mode(attributes.mode);
+
+ if (is_activation_bwd) {
+ // Backward mode: IN_0 is dy, IN_1 is x, OUT_0 is dx
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(IN_0, Pointwise_attributes::input_names::IN_0);
+ auto dy_desc = tensors.at(IN_0->second->get_uid())->get_raw_desc();
+
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(IN_1, Pointwise_attributes::input_names::IN_1);
+ auto x_desc = tensors.at(IN_1->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_POINTWISE_XDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &x_desc));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_POINTWISE_DYDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &dy_desc));
+
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(OUT_0, Pointwise_attributes::output_names::OUT_0);
+ auto dx_desc = tensors.at(OUT_0->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_POINTWISE_DXDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &dx_desc));
+ } else {
+ // Forward mode
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(IN_0, Pointwise_attributes::input_names::IN_0);
+ auto x_desc = tensors.at(IN_0->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_POINTWISE_XDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &x_desc));
+
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(OUT_0, Pointwise_attributes::output_names::OUT_0);
+ auto y_desc = tensors.at(OUT_0->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_POINTWISE_YDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &y_desc));
+
+ if (port_count >= 3) {
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(IN_1, Pointwise_attributes::input_names::IN_1);
+ auto b_desc = tensors.at(IN_1->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_POINTWISE_BDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &b_desc));
+ }
+
+ if (port_count >= 4) {
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(IN_2, Pointwise_attributes::input_names::IN_2);
+ auto t_desc = tensors.at(IN_2->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_POINTWISE_TDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &t_desc));
+ }
+ }
+
+ // Set alpha scaling factors (always set to 1.0)
+ float alpha1 = 1.0f;
+ float alpha2 = 1.0f;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(
+ pointwise_operation.get_raw_desc(), CUDNN_ATTR_OPERATION_POINTWISE_ALPHA1, CUDNN_TYPE_FLOAT, 1, &alpha1));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(
+ pointwise_operation.get_raw_desc(), CUDNN_ATTR_OPERATION_POINTWISE_ALPHA2, CUDNN_TYPE_FLOAT, 1, &alpha2));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(pointwise_operation.get_raw_desc()));
+
+ operations.push_back(std::make_shared<Operation_v8>(std::move(pointwise_operation)));
+
+ auto const& non_virtual_uids = attributes.get_non_virtual_uids();
+ uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end());
+ return {error_code_t::OK, ""};
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ j.update(R"({"tag": "POINTWISE"})"_json);
+ }
+#endif
+};
+
+inline void
+INode::pointwise(std::shared_ptr<Tensor_attributes> a,
+ Pointwise_attributes attributes,
+ std::shared_ptr<Tensor_attributes> c) {
+ attributes.inputs[Pointwise_attributes::input_names::IN_0] = a;
+ attributes.outputs[Pointwise_attributes::output_names::OUT_0] = c;
+ sub_nodes.emplace_back(std::make_unique<PointwiseNode>(std::move(attributes), context));
+}
+
+inline void
+INode::pointwise(std::shared_ptr<Tensor_attributes> a,
+ std::shared_ptr<Tensor_attributes> b,
+ Pointwise_attributes attributes,
+ std::shared_ptr<Tensor_attributes> c) {
+ attributes.inputs[Pointwise_attributes::input_names::IN_0] = a;
+ attributes.inputs[Pointwise_attributes::input_names::IN_1] = b;
+ attributes.outputs[Pointwise_attributes::output_names::OUT_0] = c;
+ sub_nodes.emplace_back(std::make_unique<PointwiseNode>(std::move(attributes), context));
+}
+
+inline void
+INode::pointwise(std::shared_ptr<Tensor_attributes> a,
+ std::shared_ptr<Tensor_attributes> b,
+ std::shared_ptr<Tensor_attributes> c,
+ Pointwise_attributes attributes,
+ std::shared_ptr<Tensor_attributes> d) {
+ attributes.inputs[Pointwise_attributes::input_names::IN_0] = a;
+ attributes.inputs[Pointwise_attributes::input_names::IN_1] = b;
+ attributes.inputs[Pointwise_attributes::input_names::IN_2] = c;
+ attributes.outputs[Pointwise_attributes::output_names::OUT_0] = d;
+ sub_nodes.emplace_back(std::make_unique<PointwiseNode>(std::move(attributes), context));
+}
+
+inline std::shared_ptr<Tensor_attributes>
+INode::pointwise(std::shared_ptr<Tensor_attributes> a, Pointwise_attributes attributes) {
+ if (attributes.name.empty()) {
+ attributes.name += std::to_string(sub_nodes.size());
+ }
+ attributes.inputs[Pointwise_attributes::input_names::IN_0] = a;
+ if (a->get_name().empty()) {
+ a->set_name(attributes.name + "::IN_0");
+ };
+ auto OUT_0 = attributes.outputs[Pointwise_attributes::output_names::OUT_0] =
+ output_tensor(attributes.name + "::OUT_0");
+
+ sub_nodes.emplace_back(std::make_unique<PointwiseNode>(std::move(attributes), context));
+ return OUT_0;
+}
+
+inline std::shared_ptr<Tensor_attributes>
+INode::pointwise(std::shared_ptr<Tensor_attributes> a,
+ std::shared_ptr<Tensor_attributes> b,
+ Pointwise_attributes attributes) {
+ if (attributes.name.empty()) {
+ attributes.name += std::to_string(sub_nodes.size());
+ }
+ attributes.inputs[Pointwise_attributes::input_names::IN_0] = a;
+ attributes.inputs[Pointwise_attributes::input_names::IN_1] = b;
+ if (a->get_name().empty()) {
+ a->set_name(attributes.name + "::IN_0");
+ };
+ if (b->get_name().empty()) {
+ b->set_name(attributes.name + "::IN_1");
+ };
+ auto OUT_0 = attributes.outputs[Pointwise_attributes::output_names::OUT_0] =
+ output_tensor(attributes.name + "::OUT_0");
+
+ sub_nodes.emplace_back(std::make_unique<PointwiseNode>(std::move(attributes), context));
+ return OUT_0;
+}
+
+inline std::shared_ptr<Tensor_attributes>
+INode::pointwise(std::shared_ptr<Tensor_attributes> a,
+ std::shared_ptr<Tensor_attributes> b,
+ std::shared_ptr<Tensor_attributes> c,
+ Pointwise_attributes attributes) {
+ if (attributes.name.empty()) {
+ attributes.name += std::to_string(sub_nodes.size());
+ }
+ attributes.inputs[Pointwise_attributes::input_names::IN_0] = a;
+ attributes.inputs[Pointwise_attributes::input_names::IN_1] = b;
+ attributes.inputs[Pointwise_attributes::input_names::IN_2] = c;
+ if (a->get_name().empty()) {
+ a->set_name(attributes.name + "::IN_0");
+ };
+ if (b->get_name().empty()) {
+ b->set_name(attributes.name + "::IN_1");
+ };
+ if (c->get_name().empty()) {
+ c->set_name(attributes.name + "::IN_2");
+ };
+ auto OUT_0 = attributes.outputs[Pointwise_attributes::output_names::OUT_0] =
+ output_tensor(attributes.name + "::OUT_0");
+
+ sub_nodes.emplace_back(std::make_unique<PointwiseNode>(std::move(attributes), context));
+ return OUT_0;
+}
+} // namespace cudnn_frontend::graph \ No newline at end of file
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/reduction.h b/deps/cudnn-frontend/include/cudnn_frontend/node/reduction.h
new file mode 100644
index 0000000..5944b69
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/node/reduction.h
@@ -0,0 +1,232 @@
+#pragma once
+
+#include "../graph_helpers.h"
+#include "../node_interface.h"
+
+namespace cudnn_frontend::graph {
+
+class ReductionNode : public NodeCRTP<ReductionNode> {
+ public:
+ Reduction_attributes attributes;
+
+ ReductionNode(Reduction_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP(context), attributes(std::move(attributes_)) {}
+
+ Type
+ getType() override final {
+ return Type::REDUCTION;
+ }
+
+ error_t
+ pre_validate_node() const override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating ReductionNode " << attributes.name);
+
+ if (attributes.get_is_deterministic() && detail::get_backend_version() < 91100) {
+ return {error_code_t::GRAPH_NOT_SUPPORTED, "DETERMINISTIC mode is not supported in cudnn version < 9.11.0"};
+ }
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ infer_properties_node() override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for reduction node " << attributes.name);
+
+ attributes.fill_from_context(context);
+
+ // Only inferrencing from IN_0 to OUT_0 works today.
+ auto x_tensor = attributes.inputs[Reduction_attributes::input_names::X];
+ auto y_tensor = attributes.outputs[Reduction_attributes::output_names::Y];
+
+ auto const& x_tensor_dim = x_tensor->get_dim();
+ auto y_tensor_dim = y_tensor->get_dim();
+ // Only infer dims and strides if user did not set them
+ if (y_tensor_dim.empty()) {
+ y_tensor->set_dim(x_tensor_dim);
+ }
+ if (y_tensor->get_stride().empty()) {
+ auto const& y_dim = y_tensor->get_dim();
+ // Default to NHWC
+ auto const& stride_order = detail::generate_NHWC_stride_order(y_dim.size());
+ y_tensor->set_stride(detail::generate_stride(y_dim, stride_order));
+ }
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ create_cudnn_operations(
+ std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations,
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations,
+ managed_backend_descriptor_t& raw_operations,
+ std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final {
+ CUDNN_FRONTEND_UNUSED(raw_operations);
+ CUDNN_FE_LOG_LABEL("INFO: " << "Building ReductionNode operations " << attributes.name << " ");
+
+ // Create reduction descriptor by directly calling cuDNN backend API
+ ReductionDesc_v8 reduction_descriptor;
+
+ // 1. Create the backend descriptor
+
+ _CUDNN_CHECK_CUDNN_ERROR(
+ reduction_descriptor.initialize_managed_backend_pointer(CUDNN_BACKEND_REDUCTION_DESCRIPTOR));
+
+ // 2. Set compute type attribute
+ cudnnDataType_t cudnn_data_type;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.compute_data_type, cudnn_data_type));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(reduction_descriptor.get_raw_desc(),
+ CUDNN_ATTR_REDUCTION_COMP_TYPE,
+ CUDNN_TYPE_DATA_TYPE,
+ 1,
+ &cudnn_data_type));
+
+ // 3. Set reduction operator attribute
+ cudnnReduceTensorOp_t cudnn_reduction_mode;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.get_mode().value(), cudnn_reduction_mode));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(reduction_descriptor.get_raw_desc(),
+ CUDNN_ATTR_REDUCTION_OPERATOR,
+ CUDNN_TYPE_REDUCTION_OPERATOR_TYPE,
+ 1,
+ &cudnn_reduction_mode));
+
+ // 4. Set deterministic mode if supported
+#if (CUDNN_VERSION >= 91100)
+ if (detail::get_backend_version() >= 91100) {
+ bool is_deterministic = attributes.get_is_deterministic();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(reduction_descriptor.get_raw_desc(),
+ CUDNN_ATTR_REDUCTION_IS_DETERMINISTIC,
+ CUDNN_TYPE_BOOLEAN,
+ 1,
+ &is_deterministic));
+ }
+#endif
+
+ // 5. Finalize the descriptor
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(reduction_descriptor.get_raw_desc()));
+ CUDNN_FE_LOG_LABEL_ENDL(reduction_descriptor);
+
+ // Create operation by directly calling cuDNN backend API
+ Operation_v8 reduction_operation;
+
+ // Validate input tensors are set
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Reduction_attributes::input_names::X);
+ auto Group_offset = attributes.inputs.find(Reduction_attributes::input_names::Group_offset);
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(Y, Reduction_attributes::output_names::Y);
+
+ // 1. Create the backend operation descriptor
+
+ _CUDNN_CHECK_CUDNN_ERROR(
+ reduction_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_REDUCTION_DESCRIPTOR));
+
+ // 2. Set the reduction descriptor attribute
+ auto reduction_desc_ptr = reduction_descriptor.get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(reduction_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_REDUCTION_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &reduction_desc_ptr));
+
+ // 3. Set the input tensor (X) descriptor attribute
+ auto x_backend_desc = tensors.at(X->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(reduction_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_REDUCTION_XDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &x_backend_desc));
+
+ // 4. Set the output tensor (Y) descriptor attribute
+ auto y_backend_desc = tensors.at(Y->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(reduction_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_REDUCTION_YDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &y_backend_desc));
+
+ // 5. Set the optional group offset tensor descriptor
+ if (Group_offset != attributes.inputs.end() && Group_offset->second != nullptr) {
+#if (CUDNN_VERSION >= 92400)
+ RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 92400,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Reduction group_offset is not supported in cudnn version < 9.24.0");
+ auto group_offset_backend_desc = tensors.at(Group_offset->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(reduction_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_REDUCTION_GROUP_OFFSET_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &group_offset_backend_desc));
+#else
+ return {error_code_t::GRAPH_NOT_SUPPORTED,
+ "Reduction group_offset is not supported in cudnn version < 9.24.0"};
+#endif
+ }
+
+ // 6. Finalize the operation descriptor
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(reduction_operation.get_raw_desc()));
+
+ operations.push_back(std::make_shared<Operation_v8>(std::move(reduction_operation)));
+
+ auto const& non_virtual_uids = attributes.get_non_virtual_uids();
+ uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end());
+ return {error_code_t::OK, ""};
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ j.update(R"({"tag": "REDUCTION"})"_json);
+ }
+#endif
+};
+
+inline void
+INode::reduction(std::shared_ptr<Tensor_attributes> a,
+ Reduction_attributes attributes,
+ std::shared_ptr<Tensor_attributes> c) {
+ attributes.inputs[Reduction_attributes::input_names::X] = a;
+ attributes.outputs[Reduction_attributes::output_names::Y] = c;
+ sub_nodes.emplace_back(std::make_unique<ReductionNode>(std::move(attributes), context));
+}
+
+inline void
+INode::reduction(std::shared_ptr<Tensor_attributes> a,
+ std::shared_ptr<Tensor_attributes> group_offset,
+ Reduction_attributes attributes,
+ std::shared_ptr<Tensor_attributes> c) {
+ attributes.inputs[Reduction_attributes::input_names::X] = a;
+ attributes.inputs[Reduction_attributes::input_names::Group_offset] = group_offset;
+ attributes.outputs[Reduction_attributes::output_names::Y] = c;
+ sub_nodes.emplace_back(std::make_unique<ReductionNode>(std::move(attributes), context));
+}
+
+inline std::shared_ptr<Tensor_attributes>
+INode::reduction(std::shared_ptr<Tensor_attributes> input, Reduction_attributes attributes) {
+ attributes.inputs[Reduction_attributes::input_names::X] = input;
+ auto Y = attributes.outputs[Reduction_attributes::output_names::Y] = output_tensor(attributes.name + "::Y");
+
+ sub_nodes.emplace_back(std::make_unique<ReductionNode>(std::move(attributes), context));
+ return Y;
+}
+
+inline std::shared_ptr<Tensor_attributes>
+INode::reduction(std::shared_ptr<Tensor_attributes> input,
+ std::shared_ptr<Tensor_attributes> group_offset,
+ Reduction_attributes attributes) {
+ attributes.inputs[Reduction_attributes::input_names::X] = input;
+ attributes.inputs[Reduction_attributes::input_names::Group_offset] = group_offset;
+ auto Y = attributes.outputs[Reduction_attributes::output_names::Y] = output_tensor(attributes.name + "::Y");
+
+ sub_nodes.emplace_back(std::make_unique<ReductionNode>(std::move(attributes), context));
+ return Y;
+}
+} // namespace cudnn_frontend::graph
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/resample.h b/deps/cudnn-frontend/include/cudnn_frontend/node/resample.h
new file mode 100644
index 0000000..34e6031
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/node/resample.h
@@ -0,0 +1,291 @@
+#pragma once
+
+#include "../graph_helpers.h"
+#include "../node_interface.h"
+
+namespace cudnn_frontend::graph {
+
+class ResampleNode : public NodeCRTP<ResampleNode> {
+ public:
+ Resample_attributes attributes;
+
+ ResampleNode(Resample_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP(context), attributes(std::move(attributes_)) {}
+
+ Type
+ getType() override final {
+ return Type::RESAMPLE;
+ }
+
+ error_t
+ pre_validate_node() const override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: " << "Validating ResampleNode " << attributes.name);
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.generate_index.has_value() == false,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "generate_index attribute not set");
+
+ if (attributes.generate_index.value() == true && attributes.resample_mode == ResampleMode_t::MAXPOOL) {
+ CUDNN_FE_VALIDATE_OUTPUT_TENSOR(Resample_attributes::output_names::Index);
+ }
+
+ // Make sure that the mode can be lowered to BE
+ cudnnResampleMode_t dummy;
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ detail::convert_to_cudnn_type(attributes.resample_mode, dummy) != CUDNN_STATUS_SUCCESS,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "Invalid resample mode.");
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ infer_properties_node() override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for resample node " << attributes.name);
+
+ auto y_tensor = attributes.outputs[Resample_attributes::output_names::Y];
+ auto x_tensor = attributes.inputs[Resample_attributes::input_names::X];
+
+ attributes.fill_from_context(context);
+
+ // If user does not set shape and layout of the output tensor,
+ // Get it from node attributes
+ if (y_tensor->get_dim().empty()) {
+ auto const x_dim = x_tensor->get_dim();
+ auto y_dim = y_tensor->get_dim();
+ y_dim = x_dim;
+
+ // 2 cause first two dimensions are batch and channels
+ for (auto dim = 2u; dim < x_dim.size(); ++dim) {
+ auto spatial_dim = dim - 2u;
+ y_dim[dim] =
+ 1 + (x_dim[dim] + attributes.pre_padding[spatial_dim].numerator +
+ attributes.post_padding[spatial_dim].numerator - attributes.window[spatial_dim].numerator) /
+ attributes.stride[spatial_dim].numerator;
+ }
+
+ y_tensor->set_dim(y_dim);
+ }
+
+ // If layout is not set, generate the strides from layout
+ if (y_tensor->get_stride().empty()) {
+ auto const& y_dim = y_tensor->get_dim();
+ // Default to NHWC
+ auto const& stride_order = detail::generate_NHWC_stride_order(y_dim.size());
+ y_tensor->set_stride(detail::generate_stride(y_dim, stride_order));
+ }
+
+ if (attributes.outputs[Resample_attributes::output_names::Index]) {
+ auto index_tensor = attributes.outputs[Resample_attributes::output_names::Index];
+ index_tensor->set_dim(y_tensor->get_dim());
+
+ // If layout is not set, generate the strides from layout
+ if (index_tensor->get_stride().empty()) {
+ auto const& index_dim = index_tensor->get_dim();
+ // Default to NHWC
+ auto const& stride_order = detail::generate_NHWC_stride_order(index_dim.size());
+ index_tensor->set_stride(detail::generate_stride(index_dim, stride_order));
+ }
+ }
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ create_cudnn_operations(
+ std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations,
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations,
+ managed_backend_descriptor_t& raw_operations,
+ std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final {
+ CUDNN_FRONTEND_UNUSED(raw_operations);
+ CUDNN_FE_LOG_LABEL("INFO: " << "Building ResampleNode operations " << attributes.name << " ");
+
+ auto number_of_spatial_dim = static_cast<int64_t>(attributes.window.size());
+
+ // Create resample descriptor by directly calling cuDNN backend API
+ ResampleDesc_v8 resample_descriptor;
+
+ _CUDNN_CHECK_CUDNN_ERROR(
+ resample_descriptor.initialize_managed_backend_pointer(CUDNN_BACKEND_RESAMPLE_DESCRIPTOR));
+
+ // Set resample mode
+ cudnnResampleMode_t cudnn_resample_mode;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.resample_mode, cudnn_resample_mode));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_descriptor.get_raw_desc(),
+ CUDNN_ATTR_RESAMPLE_MODE,
+ CUDNN_TYPE_RESAMPLE_MODE,
+ 1,
+ &cudnn_resample_mode));
+
+ // Set compute type
+ cudnnDataType_t cudnn_data_type;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.compute_data_type, cudnn_data_type));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_descriptor.get_raw_desc(),
+ CUDNN_ATTR_RESAMPLE_COMP_TYPE,
+ CUDNN_TYPE_DATA_TYPE,
+ 1,
+ &cudnn_data_type));
+
+ // Set nan propagation
+ cudnnNanPropagation_t nan_opt = CUDNN_PROPAGATE_NAN;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_descriptor.get_raw_desc(),
+ CUDNN_ATTR_RESAMPLE_NAN_PROPAGATION,
+ CUDNN_TYPE_NAN_PROPOGATION,
+ 1,
+ &nan_opt));
+
+ // Set padding mode
+ cudnnPaddingMode_t cudnn_padding_mode;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.padding_mode, cudnn_padding_mode));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_descriptor.get_raw_desc(),
+ CUDNN_ATTR_RESAMPLE_PADDING_MODE,
+ CUDNN_TYPE_PADDING_MODE,
+ 1,
+ &cudnn_padding_mode));
+
+ // Set spatial dimensions
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_descriptor.get_raw_desc(),
+ CUDNN_ATTR_RESAMPLE_SPATIAL_DIMS,
+ CUDNN_TYPE_INT64,
+ 1,
+ &number_of_spatial_dim));
+
+ // Set window dimensions
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_descriptor.get_raw_desc(),
+ CUDNN_ATTR_RESAMPLE_WINDOW_DIMS,
+ CUDNN_TYPE_FRACTION,
+ number_of_spatial_dim,
+ attributes.window.data()));
+
+ // Set pre padding
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_descriptor.get_raw_desc(),
+ CUDNN_ATTR_RESAMPLE_PRE_PADDINGS,
+ CUDNN_TYPE_FRACTION,
+ number_of_spatial_dim,
+ attributes.pre_padding.data()));
+
+ // Set post padding
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_descriptor.get_raw_desc(),
+ CUDNN_ATTR_RESAMPLE_POST_PADDINGS,
+ CUDNN_TYPE_FRACTION,
+ number_of_spatial_dim,
+ attributes.post_padding.data()));
+
+ // Set strides
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_descriptor.get_raw_desc(),
+ CUDNN_ATTR_RESAMPLE_STRIDES,
+ CUDNN_TYPE_FRACTION,
+ number_of_spatial_dim,
+ attributes.stride.data()));
+
+ // Finalize the descriptor
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(resample_descriptor.get_raw_desc()));
+ CUDNN_FE_LOG_LABEL_ENDL(resample_descriptor);
+
+ // Create operation by directly calling cuDNN backend API
+ Operation_v8 resample_operation;
+
+ _CUDNN_CHECK_CUDNN_ERROR(
+ resample_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_RESAMPLE_FWD_DESCRIPTOR));
+
+ // Set input tensor X
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Resample_attributes::input_names::X);
+ auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_RESAMPLE_FWD_XDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &x_desc));
+
+ // Set output tensor Y
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(Y, Resample_attributes::output_names::Y);
+ auto y_desc = tensors.at(Y->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_RESAMPLE_FWD_YDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &y_desc));
+
+ // Set alpha and beta
+ double alpha = 1.0;
+ double beta = 0.0;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(
+ resample_operation.get_raw_desc(), CUDNN_ATTR_OPERATION_RESAMPLE_FWD_ALPHA, CUDNN_TYPE_DOUBLE, 1, &alpha));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(
+ resample_operation.get_raw_desc(), CUDNN_ATTR_OPERATION_RESAMPLE_FWD_BETA, CUDNN_TYPE_DOUBLE, 1, &beta));
+
+ // Set resample descriptor
+ auto resample_raw_desc = resample_descriptor.get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_RESAMPLE_FWD_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &resample_raw_desc));
+
+ // Set index tensor if available
+ auto index = attributes.outputs.find(Resample_attributes::output_names::Index);
+ if ((index != attributes.outputs.end()) && (index->second != nullptr)) {
+ auto idx_desc = tensors.at(index->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_RESAMPLE_FWD_IDXDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &idx_desc));
+ }
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(resample_operation.get_raw_desc()));
+
+ operations.push_back(std::make_shared<Operation_v8>(std::move(resample_operation)));
+
+ auto const& non_virtual_uids = attributes.get_non_virtual_uids();
+ uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end());
+ return {error_code_t::OK, ""};
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ j.update(R"( {"tag": "RESAMPLE"})"_json);
+ }
+#endif
+};
+
+inline std::array<std::shared_ptr<Tensor_attributes>, 2>
+INode::resample(std::shared_ptr<Tensor_attributes> input, Resample_attributes attributes) {
+ if (attributes.name.empty()) {
+ attributes.name += std::to_string(sub_nodes.size());
+ }
+ attributes.inputs[Resample_attributes::input_names::X] = input;
+ auto Y = attributes.outputs[Resample_attributes::output_names::Y] = output_tensor(attributes.name + "::Y");
+ std::shared_ptr<Tensor_attributes> Index = nullptr;
+ if (attributes.generate_index.has_value() && attributes.generate_index.value() == true &&
+ attributes.resample_mode == ResampleMode_t::MAXPOOL) {
+ Index = attributes.outputs[Resample_attributes::output_names::Index] =
+ output_tensor(attributes.name + "::Index");
+ }
+
+ sub_nodes.emplace_back(std::make_unique<ResampleNode>(std::move(attributes), context));
+ return {Y, Index};
+}
+
+} // namespace cudnn_frontend::graph \ No newline at end of file
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/reshape.h b/deps/cudnn-frontend/include/cudnn_frontend/node/reshape.h
new file mode 100644
index 0000000..e2b5546
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/node/reshape.h
@@ -0,0 +1,156 @@
+#pragma once
+
+#include "../graph_helpers.h"
+#include "../node_interface.h"
+
+namespace cudnn_frontend::graph {
+
+class ReshapeNode : public NodeCRTP<ReshapeNode> {
+ public:
+ Reshape_attributes attributes;
+
+ ReshapeNode(Reshape_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP(context), attributes(std::move(attributes_)) {}
+
+ Type
+ getType() override final {
+ return Type::RESHAPE;
+ }
+
+ error_t
+ infer_properties_node() override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for reshape node " << attributes.name);
+
+ auto y_tensor = attributes.outputs[Reshape_attributes::output_names::Y];
+
+ attributes.fill_from_context(context);
+
+ // If user does not set shape and layout of the output tensor,
+ // Get it from node attributes
+ // If layout is not set, generate the strides from layout
+
+ if (y_tensor->get_dim().empty() && attributes.get_dim().size()) {
+ y_tensor->set_dim(attributes.dim);
+ }
+
+ // Graph producers may represent scalar reshape outputs
+ // as rank-0 tensors (dim={}). Without promotion, stride inference below
+ // would call generate_NHWC_stride_order(0), which is UB, and the cuDNN
+ // backend rejects rank-0 descriptors regardless. Promote to canonical
+ // rank-1 length-1 -- value-preserving because a scalar has volume 1 --
+ // while keeping the normalization node-local so downstream broadcast/shape
+ // inference remains intact.
+ if (y_tensor->get_dim().empty()) {
+ y_tensor->set_dim({1});
+ if (y_tensor->get_stride().empty()) {
+ y_tensor->set_stride({1});
+ }
+ }
+
+ if (y_tensor->get_stride().empty()) {
+ if (attributes.get_stride().size()) {
+ y_tensor->set_stride(attributes.get_stride());
+ } else {
+ auto const& y_dim = y_tensor->get_dim();
+ // Default to NHWC for multi-axis tensors. generate_NHWC_stride_order
+ // indexes stride_order[1] and assumes num_dims >= 2; use row-major
+ // for scalars (0-D) and vectors (1-D).
+ const int64_t rank = static_cast<int64_t>(y_dim.size());
+ std::vector<int64_t> const stride_order =
+ rank < 2 ? detail::generate_row_major_stride_order(rank) : detail::generate_NHWC_stride_order(rank);
+ y_tensor->set_stride(detail::generate_stride(y_dim, stride_order));
+ }
+ }
+
+ if (y_tensor->get_dim().empty() || y_tensor->get_stride().empty()) {
+ return {error_code_t::SHAPE_DEDUCTION_FAILED, "Reshape node output shape deduction failed"};
+ }
+
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Reshape_attributes::input_names::X);
+ auto const& input_data_type = X->second->get_data_type();
+ if (y_tensor->get_data_type() == DataType_t::NOT_SET) {
+ y_tensor->set_data_type(input_data_type);
+ } else if (attributes.get_reshape_mode() == ReshapeMode_t::LOGICAL) {
+ // Lexicographic reshape preserves element type; reject inconsistent metadata.
+ // VIEW_ONLY paths (e.g. SDPA backward) may set Y dtype after reshape to match a consumer.
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ y_tensor->get_data_type() != input_data_type,
+ error_code_t::INVALID_VALUE,
+ "Output and input tensor data types must match for LOGICAL reshape operation.");
+ }
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ create_cudnn_operations(
+ std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations,
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations,
+ managed_backend_descriptor_t& raw_operations,
+ std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final {
+ CUDNN_FRONTEND_UNUSED(raw_operations);
+ CUDNN_FE_LOG_LABEL("INFO: " << "Building ReshapeNode operations " << attributes.name << " ");
+
+ // Create operation by directly calling cuDNN backend API
+ Operation_v8 reshape_operation;
+
+ _CUDNN_CHECK_CUDNN_ERROR(
+ reshape_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_RESHAPE_DESCRIPTOR));
+
+ // Set input tensor X
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Reshape_attributes::input_names::X);
+ auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(reshape_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_RESHAPE_XDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &x_desc));
+#if (CUDNN_VERSION >= 92200)
+ // Set reshape mode
+ cudnnBackendReshapeMode_t cudnn_reshape_mode;
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.get_reshape_mode(), cudnn_reshape_mode));
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(reshape_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_RESHAPE_MODE,
+ CUDNN_TYPE_RESHAPE_MODE,
+ 1,
+ &cudnn_reshape_mode));
+#endif
+ // Set output tensor Y
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(Y, Reshape_attributes::output_names::Y);
+ auto y_desc = tensors.at(Y->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(reshape_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_RESHAPE_YDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &y_desc));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(reshape_operation.get_raw_desc()));
+
+ operations.push_back(std::make_shared<Operation_v8>(std::move(reshape_operation)));
+
+ auto const& non_virtual_uids = attributes.get_non_virtual_uids();
+ uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end());
+ return {error_code_t::OK, ""};
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ j.update(R"( {"tag": "RESHAPE"})"_json);
+ }
+#endif
+};
+
+inline std::shared_ptr<Tensor_attributes>
+INode::reshape(std::shared_ptr<Tensor_attributes> input, Reshape_attributes attributes) {
+ attributes.inputs[Reshape_attributes::input_names::X] = input;
+ auto Y = attributes.outputs[Reshape_attributes::output_names::Y] = output_tensor(attributes.name + "::Y");
+
+ sub_nodes.emplace_back(std::make_unique<ReshapeNode>(std::move(attributes), context));
+ return Y;
+}
+
+} // namespace cudnn_frontend::graph \ No newline at end of file
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/rmsnorm.h b/deps/cudnn-frontend/include/cudnn_frontend/node/rmsnorm.h
new file mode 100644
index 0000000..bc1f37d
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/node/rmsnorm.h
@@ -0,0 +1,406 @@
+#pragma once
+
+#include "../graph_helpers.h"
+#include "../node_interface.h"
+
+namespace cudnn_frontend {
+
+namespace graph {
+class RMSNormNode : public NodeCRTP<RMSNormNode> {
+ public:
+ Rmsnorm_attributes attributes;
+
+ RMSNormNode(Rmsnorm_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP(context), attributes(std::move(attributes_)) {}
+
+ Type
+ getType() override final {
+ return Type::RMSNORM;
+ }
+
+ error_t
+ infer_properties_node() override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for rmsnorm node " << attributes.name);
+
+ attributes.fill_from_context(context);
+
+ auto X = attributes.inputs[Rmsnorm_attributes::input_names::X];
+ auto Y = attributes.outputs[Rmsnorm_attributes::output_names::Y];
+
+ // Only infer dims and strides if user did not set them
+ if (Y->get_dim().empty()) {
+ Y->set_dim(X->get_dim());
+ }
+ if (Y->get_stride().empty()) {
+ Y->set_stride(X->get_stride());
+ }
+
+ if (attributes.forward_phase == NormFwdPhase_t::TRAINING) {
+ auto inv_var = attributes.outputs[Rmsnorm_attributes::output_names::INV_VARIANCE];
+ // Only infer dims and strides if user did not set them
+ if (inv_var->get_dim().empty()) {
+ auto inv_var_dim = X->get_dim();
+ auto scale = attributes.inputs[Rmsnorm_attributes::input_names::SCALE];
+ if (scale->get_dim().empty()) {
+ // mean inv_var dim is n,1,1,1
+ for (size_t i = 1; i < inv_var_dim.size(); i++) {
+ inv_var_dim[i] = 1;
+ }
+ } else {
+ for (size_t i = 0; i < inv_var_dim.size(); i++) {
+ if (scale->get_dim()[i] != 1) {
+ inv_var_dim[i] = 1;
+ }
+ }
+ }
+ inv_var->set_dim(inv_var_dim);
+ }
+ if (inv_var->get_stride().empty()) {
+ auto const& inv_var_dim = inv_var->get_dim();
+ std::vector<int64_t> stride_order;
+ CHECK_CUDNN_FRONTEND_ERROR(
+ detail::generate_stride_order_preserving_format(X->get_stride(), inv_var_dim.size(), stride_order));
+ inv_var->set_stride(detail::generate_stride(inv_var_dim, stride_order));
+ }
+ }
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ pre_validate_node() const override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating RMSNormNode " << attributes.name);
+
+ // Norm forward phase should be set
+ RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.forward_phase == NormFwdPhase_t::NOT_SET,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "Forward phase not set of rmsnorm node.");
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ create_cudnn_operations(
+ std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations,
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations,
+ managed_backend_descriptor_t& raw_operations,
+ std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final {
+ CUDNN_FRONTEND_UNUSED(raw_operations);
+ CUDNN_FE_LOG_LABEL("INFO: Building RMSNormNode operations " << attributes.name << " ");
+
+ // Create operation by directly calling cuDNN backend API
+ Operation_v8 rmsnorm_operation;
+
+ _CUDNN_CHECK_CUDNN_ERROR(
+ rmsnorm_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_NORM_FORWARD_DESCRIPTOR));
+
+ // Set norm mode to RMS_NORM
+ cudnnBackendNormMode_t cudnn_norm_mode;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormMode_t::RMS_NORM, cudnn_norm_mode));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rmsnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_MODE,
+ CUDNN_TYPE_NORM_MODE,
+ 1,
+ &cudnn_norm_mode));
+
+ // Set forward phase
+ cudnnBackendNormFwdPhase_t cudnn_norm_fwd_phase;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.forward_phase, cudnn_norm_fwd_phase));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rmsnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_PHASE,
+ CUDNN_TYPE_NORM_FWD_PHASE,
+ 1,
+ &cudnn_norm_fwd_phase));
+
+ // Set input tensor X
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Rmsnorm_attributes::input_names::X);
+ auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rmsnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_XDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &x_desc));
+
+ // Set scale tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SCALE, Rmsnorm_attributes::input_names::SCALE);
+ auto scale_desc = tensors.at(SCALE->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rmsnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_SCALE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &scale_desc));
+
+ // Set epsilon tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(EPSILON, Rmsnorm_attributes::input_names::EPSILON);
+ auto epsilon_desc = tensors.at(EPSILON->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rmsnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_EPSILON_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &epsilon_desc));
+
+ // Set output tensor Y
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(Y, Rmsnorm_attributes::output_names::Y);
+ auto y_desc = tensors.at(Y->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rmsnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_YDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &y_desc));
+
+ // Set inv_variance for training phase
+ if (attributes.forward_phase == NormFwdPhase_t::TRAINING) {
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(INV_VARIANCE, Rmsnorm_attributes::output_names::INV_VARIANCE);
+ auto inv_var_desc = tensors.at(INV_VARIANCE->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rmsnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_INV_VARIANCE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &inv_var_desc));
+ }
+
+ // Set optional bias tensor
+ auto BIAS = attributes.inputs.find(Rmsnorm_attributes::input_names::BIAS);
+ if ((BIAS != attributes.inputs.end()) && (BIAS->second != nullptr)) {
+ auto bias_desc = tensors.at(BIAS->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rmsnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_BIAS_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &bias_desc));
+ }
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(rmsnorm_operation.get_raw_desc()));
+
+ operations.push_back(std::make_shared<Operation_v8>(std::move(rmsnorm_operation)));
+
+ auto const& non_virtual_uids = attributes.get_non_virtual_uids();
+ uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end());
+ return {error_code_t::OK, ""};
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ j.update(R"( {"tag": "RMS_NORM"})"_json);
+ }
+#endif
+};
+
+class DRMSNormNode : public NodeCRTP<DRMSNormNode> {
+ public:
+ Rmsnorm_backward_attributes attributes;
+
+ DRMSNormNode(Rmsnorm_backward_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP(context), attributes(std::move(attributes_)) {}
+
+ Type
+ getType() override final {
+ return Type::DRMSNorm;
+ }
+
+ error_t
+ pre_validate_node() const override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating DRMSNormNode node " << attributes.name);
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.use_dbias.has_value() == false,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "DRMSNormNode node needs has_bias(bool) to be called.");
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ infer_properties_node() override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for DRMSNorm node " << attributes.name);
+
+ attributes.fill_from_context(context);
+
+ // TODO: Only inferencing from X works today.
+ auto X = attributes.inputs[Rmsnorm_backward_attributes::input_names::X];
+ auto const x_tensor_dim = X->get_dim();
+
+ auto DY = attributes.inputs[Rmsnorm_backward_attributes::input_names::DY];
+ auto dy_tensor_dim = DY->get_dim();
+
+ // Only infer dims and strides if user did not set them
+ if (dy_tensor_dim.empty()) {
+ dy_tensor_dim.resize(x_tensor_dim.size());
+ DY->set_dim(x_tensor_dim);
+ }
+ if (DY->get_stride().empty()) {
+ auto const& DY_dim = DY->get_dim();
+ // Default to NHWC
+ auto const& stride_order = detail::generate_NHWC_stride_order(DY_dim.size());
+ DY->set_stride(detail::generate_stride(DY_dim, stride_order));
+ }
+
+ auto DX = attributes.outputs[Rmsnorm_backward_attributes::output_names::DX];
+ auto dx_tensor_dim = DX->get_dim();
+ // Only infer dims and strides if user did not set them
+ if (dx_tensor_dim.empty()) {
+ dx_tensor_dim.resize(x_tensor_dim.size());
+ DX->set_dim(x_tensor_dim);
+ }
+ if (DX->get_stride().empty()) {
+ auto const& DX_dim = DX->get_dim();
+ // Default to NHWC
+ auto const& stride_order = detail::generate_NHWC_stride_order(DX_dim.size());
+ DX->set_stride(detail::generate_stride(DX_dim, stride_order));
+ }
+
+ auto scale_bias_dim = X->get_dim();
+ scale_bias_dim[0] = 1;
+
+ // Set channel length tensors
+ auto infer_scale_bias_tensors = [&scale_bias_dim](std::shared_ptr<Tensor_attributes>& T) {
+ auto tensor_dim = T->get_dim();
+ // Only infer dims and strides if user did not set them
+ if (tensor_dim.empty()) {
+ T->set_dim(scale_bias_dim);
+ }
+ if (T->get_stride().empty()) {
+ auto const& T_dim = T->get_dim();
+ // Default to NHWC
+ auto const& stride_order = detail::generate_NHWC_stride_order(T_dim.size());
+ T->set_stride(detail::generate_stride(T_dim, stride_order));
+ }
+ };
+
+ infer_scale_bias_tensors(attributes.outputs[Rmsnorm_backward_attributes::output_names::DSCALE]);
+ if (attributes.use_dbias.value()) {
+ infer_scale_bias_tensors(attributes.outputs[Rmsnorm_backward_attributes::output_names::DBIAS]);
+ }
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ create_cudnn_operations(
+ std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations,
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations,
+ managed_backend_descriptor_t& raw_operations,
+ std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final {
+ CUDNN_FRONTEND_UNUSED(raw_operations);
+ CUDNN_FE_LOG_LABEL("INFO: Building DRMSNormNode operations " << attributes.name << " ");
+
+ // Create operation by directly calling cuDNN backend API
+ Operation_v8 drmsnorm_operation;
+
+ _CUDNN_CHECK_CUDNN_ERROR(
+ drmsnorm_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_NORM_BACKWARD_DESCRIPTOR));
+
+ // Set norm mode to RMS_NORM
+ cudnnBackendNormMode_t cudnn_norm_mode;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormMode_t::RMS_NORM, cudnn_norm_mode));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(drmsnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_MODE,
+ CUDNN_TYPE_NORM_MODE,
+ 1,
+ &cudnn_norm_mode));
+
+ // Set input tensor X
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Rmsnorm_backward_attributes::input_names::X);
+ auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(drmsnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_XDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &x_desc));
+
+ // Set DY tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(DY, Rmsnorm_backward_attributes::input_names::DY);
+ auto dy_desc = tensors.at(DY->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(drmsnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_DYDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &dy_desc));
+
+ // Set scale tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SCALE, Rmsnorm_backward_attributes::input_names::SCALE);
+ auto scale_desc = tensors.at(SCALE->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(drmsnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_SCALE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &scale_desc));
+
+ // Set inv_variance tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(INV_VARIANCE, Rmsnorm_backward_attributes::input_names::INV_VARIANCE);
+ auto inv_var_desc = tensors.at(INV_VARIANCE->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(drmsnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_INV_VARIANCE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &inv_var_desc));
+
+ // Set DSCALE output tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DSCALE, Rmsnorm_backward_attributes::output_names::DSCALE);
+ auto dscale_desc = tensors.at(DSCALE->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(drmsnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_DSCALE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &dscale_desc));
+
+ // Set optional DBIAS output tensor
+ if (attributes.use_dbias.value()) {
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DBIAS, Rmsnorm_backward_attributes::output_names::DBIAS);
+ auto dbias_desc = tensors.at(DBIAS->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(drmsnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_DBIAS_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &dbias_desc));
+ }
+
+ // Set DX output tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DX, Rmsnorm_backward_attributes::output_names::DX);
+ auto dx_desc = tensors.at(DX->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(drmsnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_BWD_DXDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &dx_desc));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(drmsnorm_operation.get_raw_desc()));
+
+ operations.push_back(std::make_shared<Operation_v8>(std::move(drmsnorm_operation)));
+
+ auto const& non_virtual_uids = attributes.get_non_virtual_uids();
+ uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end());
+ return {error_code_t::OK, ""};
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ j.update(R"( {"tag": "RMS_NORM_BPROP"})"_json);
+ }
+#endif
+};
+
+} // namespace graph
+
+} // namespace cudnn_frontend \ No newline at end of file
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/rng.h b/deps/cudnn-frontend/include/cudnn_frontend/node/rng.h
new file mode 100644
index 0000000..09b762d
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/node/rng.h
@@ -0,0 +1,187 @@
+#pragma once
+
+#include "../graph_helpers.h"
+#include "../node_interface.h"
+
+namespace cudnn_frontend::graph {
+
+class RngNode : public NodeCRTP<RngNode> {
+ public:
+ Rng_attributes attributes;
+
+ RngNode(Rng_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP(context), attributes(std::move(attributes_)) {}
+
+ Type
+ getType() override final {
+ return Type::RNG;
+ }
+
+ error_t
+ infer_properties_node() override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for rng node " << attributes.name);
+
+ auto y_tensor = attributes.outputs[Rng_attributes::output_names::Y];
+
+ attributes.fill_from_context(context);
+
+ // If user does not set shape and layout of the generated tensor,
+ // Get it from node attributes
+ // If layout is not set, generate the strides from layout
+
+ if (y_tensor->get_dim().empty() && attributes.get_dim().size()) {
+ y_tensor->set_dim(attributes.dim);
+ }
+
+ if (y_tensor->get_stride().empty()) {
+ if (attributes.get_stride().size()) {
+ y_tensor->set_stride(attributes.get_stride());
+ } else {
+ auto const& y_dim = y_tensor->get_dim();
+ // Default to NHWC
+ auto const& stride_order = detail::generate_NHWC_stride_order(y_dim.size());
+ y_tensor->set_stride(detail::generate_stride(y_dim, stride_order));
+ }
+ }
+
+ if (y_tensor->get_dim().empty() || y_tensor->get_stride().empty()) {
+ return {error_code_t::SHAPE_DEDUCTION_FAILED, "RNG node output shape deduction failed"};
+ }
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ create_cudnn_operations(
+ std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations,
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations,
+ managed_backend_descriptor_t& raw_operations,
+ std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final {
+ CUDNN_FRONTEND_UNUSED(raw_operations);
+ CUDNN_FE_LOG_LABEL("INFO: " << "Building RngNode operations " << attributes.name << " ");
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.get_distribution() != RngDistribution_t::BERNOULLI,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "no other distribution except bernoulli supported.");
+
+ // Create RNG descriptor by directly calling cuDNN backend API
+ RngDesc_v8 rng_descriptor;
+
+ _CUDNN_CHECK_CUDNN_ERROR(rng_descriptor.initialize_managed_backend_pointer(CUDNN_BACKEND_RNG_DESCRIPTOR));
+
+ // Set distribution type
+ cudnnRngDistribution_t cudnn_rng_distribution;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.get_distribution(), cudnn_rng_distribution));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rng_descriptor.get_raw_desc(),
+ CUDNN_ATTR_RNG_DISTRIBUTION,
+ CUDNN_TYPE_RNG_DISTRIBUTION,
+ 1,
+ &cudnn_rng_distribution));
+
+ // Set Bernoulli distribution probability
+ double bernoulli_prob = attributes.get_bernoulli_probability().value();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rng_descriptor.get_raw_desc(),
+ CUDNN_ATTR_RNG_BERNOULLI_DIST_PROBABILITY,
+ CUDNN_TYPE_DOUBLE,
+ 1,
+ &bernoulli_prob));
+
+ // Finalize the descriptor
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(rng_descriptor.get_raw_desc()));
+ CUDNN_FE_LOG_LABEL_ENDL(rng_descriptor);
+
+ // Create operation by directly calling cuDNN backend API
+ Operation_v8 rng_operation;
+
+ _CUDNN_CHECK_CUDNN_ERROR(
+ rng_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_RNG_DESCRIPTOR));
+
+ // Set output tensor Y
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(Y, Rng_attributes::output_names::Y);
+ auto y_desc = tensors.at(Y->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(
+ rng_operation.get_raw_desc(), CUDNN_ATTR_OPERATION_RNG_YDESC, CUDNN_TYPE_BACKEND_DESCRIPTOR, 1, &y_desc));
+
+ // Set RNG descriptor
+ auto rng_raw_desc = rng_descriptor.get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rng_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_RNG_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &rng_raw_desc));
+
+ if (attributes.seed.has_value()) {
+ // Set seed as int64_t value
+ int64_t seed_value = attributes.get_seed().value();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(
+ rng_operation.get_raw_desc(), CUDNN_ATTR_OPERATION_RNG_SEED, CUDNN_TYPE_INT64, 1, &seed_value));
+ } else {
+ // Set seed tensor descriptor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(Seed, Rng_attributes::input_names::Seed);
+ auto seed_desc = tensors.at(Seed->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rng_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_RNG_SEED,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &seed_desc));
+
+ // Set offset tensor descriptor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(Offset, Rng_attributes::input_names::Offset);
+ auto offset_desc = tensors.at(Offset->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rng_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_RNG_OFFSET_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &offset_desc));
+ }
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(rng_operation.get_raw_desc()));
+
+ operations.push_back(std::make_shared<Operation_v8>(std::move(rng_operation)));
+
+ auto const& non_virtual_uids = attributes.get_non_virtual_uids();
+ uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end());
+ return {error_code_t::OK, ""};
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ j.update(R"( {"tag": "RNG"})"_json);
+ }
+#endif
+};
+
+inline void
+INode::rng(std::shared_ptr<Tensor_attributes> seed,
+ std::shared_ptr<Tensor_attributes> offset,
+ Rng_attributes attributes,
+ std::shared_ptr<Tensor_attributes> y) {
+ attributes.inputs[Rng_attributes::input_names::Seed] = seed;
+ attributes.inputs[Rng_attributes::input_names::Offset] = offset;
+ attributes.outputs[Rng_attributes::output_names::Y] = y;
+ sub_nodes.emplace_back(std::make_unique<RngNode>(std::move(attributes), context));
+}
+
+inline std::shared_ptr<Tensor_attributes>
+INode::rng(std::shared_ptr<Tensor_attributes> seed,
+ std::shared_ptr<Tensor_attributes> offset,
+ Rng_attributes attributes) {
+ attributes.inputs[Rng_attributes::input_names::Seed] = seed;
+ attributes.inputs[Rng_attributes::input_names::Offset] = offset;
+ auto Y = attributes.outputs[Rng_attributes::output_names::Y] = output_tensor(attributes.name + "::Y");
+
+ sub_nodes.emplace_back(std::make_unique<RngNode>(std::move(attributes), context));
+ return Y;
+}
+} // namespace cudnn_frontend::graph \ No newline at end of file
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/rope.h b/deps/cudnn-frontend/include/cudnn_frontend/node/rope.h
new file mode 100644
index 0000000..01b05c4
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/node/rope.h
@@ -0,0 +1,146 @@
+#pragma once
+
+#include "../graph_helpers.h"
+#include "../node_interface.h"
+
+namespace cudnn_frontend {
+
+namespace graph {
+class RoPENode : public NodeCRTP<RoPENode> {
+ public:
+ RoPE_attributes attributes;
+
+ RoPENode(RoPE_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP(context), attributes(std::move(attributes_)) {}
+
+ Type
+ getType() override final {
+ return Type::ROPE;
+ }
+
+ error_t
+ infer_properties_node() override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for RoPE node " << attributes.name);
+
+ attributes.fill_from_context(context);
+
+ auto INPUT = attributes.inputs[RoPE_attributes::input_names::INPUT];
+ auto OUTPUT = attributes.outputs[RoPE_attributes::output_names::OUTPUT];
+
+ if (OUTPUT->get_dim().empty()) {
+ OUTPUT->set_dim(INPUT->get_dim());
+ }
+ if (OUTPUT->get_stride().empty()) {
+ OUTPUT->set_stride(INPUT->get_stride());
+ }
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ pre_validate_node() const override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating RoPENode " << attributes.name);
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ attributes.inputs.find(RoPE_attributes::input_names::INPUT) == attributes.inputs.end(),
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "RoPE node requires INPUT tensor.");
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ attributes.inputs.find(RoPE_attributes::input_names::FREQS) == attributes.inputs.end(),
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "RoPE node requires FREQS tensor.");
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ create_cudnn_operations(
+ std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations,
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations,
+ managed_backend_descriptor_t& raw_operations,
+ std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final {
+ CUDNN_FRONTEND_UNUSED(operations);
+ CUDNN_FE_LOG_LABEL("INFO: Building RoPENode operations " << attributes.name << " ");
+
+#if (CUDNN_VERSION >= 92400)
+ // Compile- and run-time checks: RoPE op was added in cuDNN 9.24.
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ detail::get_backend_version() < 92400,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "CUDNN_BACKEND_OPERATION_ROPE_FWD_DESCRIPTOR is only available starting cuDNN 9.24.");
+
+ auto rope_operation =
+ make_shared_backend_pointer((cudnnBackendDescriptorType_t)CUDNN_BACKEND_OPERATION_ROPE_FWD_DESCRIPTOR);
+
+ // Set input X
+ auto X = attributes.inputs.find(RoPE_attributes::input_names::INPUT)->second;
+ auto backend_x = tensors[X->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rope_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_ROPE_FWD_XDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_x));
+
+ // Set freqs
+ auto FREQS = attributes.inputs.find(RoPE_attributes::input_names::FREQS)->second;
+ auto backend_freqs = tensors[FREQS->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rope_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_ROPE_FWD_FREQSDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_freqs));
+
+ // Set output Y
+ auto Y = attributes.outputs.find(RoPE_attributes::output_names::OUTPUT)->second;
+ auto backend_y = tensors[Y->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rope_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_ROPE_FWD_YDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_y));
+
+ // Set output_scale (host fp32 scalar; default 1.0)
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rope_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_ROPE_FWD_OUTPUT_SCALE,
+ CUDNN_TYPE_FLOAT,
+ 1,
+ &attributes.output_scale));
+
+ // Set rope_dim (int64; 0 = full rotation = input's head_dim)
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rope_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_ROPE_FWD_ROPE_DIM,
+ CUDNN_TYPE_INT64,
+ 1,
+ &attributes.rope_dim));
+
+ // Finalize
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(rope_operation->get_backend_descriptor()));
+
+ auto const& non_virtual_uids = attributes.get_non_virtual_uids();
+ uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end());
+
+ raw_operations.push_back(rope_operation);
+
+ return {error_code_t::OK, ""};
+#else
+ CUDNN_FRONTEND_UNUSED(uids_involved_in_operations);
+ CUDNN_FRONTEND_UNUSED(raw_operations);
+ CUDNN_FRONTEND_UNUSED(tensors);
+ return {error_code_t::GRAPH_NOT_SUPPORTED,
+ "CUDNN_BACKEND_OPERATION_ROPE_FWD_DESCRIPTOR is only available starting cuDNN 9.24."};
+#endif
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ j.update(R"({"tag": "ROPE"})"_json);
+ }
+#endif
+};
+
+} // namespace graph
+
+} // namespace cudnn_frontend
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/rope_backward.h b/deps/cudnn-frontend/include/cudnn_frontend/node/rope_backward.h
new file mode 100644
index 0000000..fd7bf4f
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/node/rope_backward.h
@@ -0,0 +1,143 @@
+#pragma once
+
+#include "../graph_helpers.h"
+#include "../node_interface.h"
+
+namespace cudnn_frontend {
+
+namespace graph {
+class RoPEBackwardNode : public NodeCRTP<RoPEBackwardNode> {
+ public:
+ RoPE_backward_attributes attributes;
+
+ RoPEBackwardNode(RoPE_backward_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP(context), attributes(std::move(attributes_)) {}
+
+ Type
+ getType() override final {
+ return Type::ROPE_BWD;
+ }
+
+ error_t
+ infer_properties_node() override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for RoPE backward node " << attributes.name);
+
+ attributes.fill_from_context(context);
+
+ auto DY = attributes.inputs[RoPE_backward_attributes::input_names::DY];
+ auto DX = attributes.outputs[RoPE_backward_attributes::output_names::DX];
+
+ if (DX->get_dim().empty()) {
+ DX->set_dim(DY->get_dim());
+ }
+ if (DX->get_stride().empty()) {
+ DX->set_stride(DY->get_stride());
+ }
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ pre_validate_node() const override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating RoPEBackwardNode " << attributes.name);
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ attributes.inputs.find(RoPE_backward_attributes::input_names::DY) == attributes.inputs.end(),
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "RoPE backward node requires DY (input gradient) tensor.");
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ attributes.inputs.find(RoPE_backward_attributes::input_names::FREQS) == attributes.inputs.end(),
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "RoPE backward node requires FREQS tensor.");
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ create_cudnn_operations(
+ std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations,
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations,
+ managed_backend_descriptor_t& raw_operations,
+ std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final {
+ CUDNN_FRONTEND_UNUSED(operations);
+ CUDNN_FE_LOG_LABEL("INFO: Building RoPEBackwardNode operations " << attributes.name << " ");
+
+#if (CUDNN_VERSION >= 92400)
+ // Compile- and run-time checks: RoPE bwd op was added in cuDNN 9.24.
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ detail::get_backend_version() < 92400,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "CUDNN_BACKEND_OPERATION_ROPE_BWD_DESCRIPTOR is only available starting cuDNN 9.24.");
+
+ auto rope_bwd_operation =
+ make_shared_backend_pointer((cudnnBackendDescriptorType_t)CUDNN_BACKEND_OPERATION_ROPE_BWD_DESCRIPTOR);
+
+ // Set DY (input gradient)
+ auto DY = attributes.inputs.find(RoPE_backward_attributes::input_names::DY)->second;
+ auto backend_dy = tensors[DY->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rope_bwd_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_ROPE_BWD_DYDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_dy));
+
+ // Set freqs
+ auto FREQS = attributes.inputs.find(RoPE_backward_attributes::input_names::FREQS)->second;
+ auto backend_freqs = tensors[FREQS->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rope_bwd_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_ROPE_BWD_FREQSDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_freqs));
+
+ // Set DX (output gradient)
+ auto DX = attributes.outputs.find(RoPE_backward_attributes::output_names::DX)->second;
+ auto backend_dx = tensors[DX->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rope_bwd_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_ROPE_BWD_DXDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_dx));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rope_bwd_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_ROPE_BWD_OUTPUT_SCALE,
+ CUDNN_TYPE_FLOAT,
+ 1,
+ &attributes.output_scale));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rope_bwd_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_ROPE_BWD_ROPE_DIM,
+ CUDNN_TYPE_INT64,
+ 1,
+ &attributes.rope_dim));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(rope_bwd_operation->get_backend_descriptor()));
+
+ auto const& non_virtual_uids = attributes.get_non_virtual_uids();
+ uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end());
+
+ raw_operations.push_back(rope_bwd_operation);
+
+ return {error_code_t::OK, ""};
+#else
+ CUDNN_FRONTEND_UNUSED(uids_involved_in_operations);
+ CUDNN_FRONTEND_UNUSED(raw_operations);
+ CUDNN_FRONTEND_UNUSED(tensors);
+ return {error_code_t::GRAPH_NOT_SUPPORTED,
+ "CUDNN_BACKEND_OPERATION_ROPE_BWD_DESCRIPTOR is only available starting cuDNN 9.24."};
+#endif
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ j.update(R"({"tag": "ROPE_BWD"})"_json);
+ }
+#endif
+};
+
+} // namespace graph
+
+} // namespace cudnn_frontend
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/scaled_dot_product_flash_attention.h b/deps/cudnn-frontend/include/cudnn_frontend/node/scaled_dot_product_flash_attention.h
new file mode 100644
index 0000000..c3c37c5
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/node/scaled_dot_product_flash_attention.h
@@ -0,0 +1,2730 @@
+
+#pragma once
+
+#include <cstdlib>
+#include <unordered_set>
+
+#include "../../cudnn_frontend_Heuristics.h"
+#include "../../cudnn_frontend_Logging.h"
+
+#include "../graph_helpers.h"
+#include "../node_interface.h"
+
+#include "diagonal_band_mask.h"
+#include "matmul.h"
+#include "pointwise.h"
+#include "reduction.h"
+#include "rng.h"
+#include "softmax.h"
+#include "paged_cache_load.h"
+#include "block_scale_dequantize.h"
+#include "sdpa_support_surface.h"
+
+namespace cudnn_frontend::graph {
+
+// If a 1-D length/offset index tensor [n] is supplied, promote it to the 4-D
+// [n, 1, 1, 1] form that the cuDNN backend requires for seq_len / cu_seq_len and
+// ragged-offset tensors. No-op if the tensor is unset or is not 1-D.
+inline void
+promote_1d_index_tensor_to_4d(std::shared_ptr<Tensor_attributes> const& tensor) {
+ if (tensor == nullptr) {
+ return;
+ }
+ auto const& dim = tensor->get_dim();
+ auto const& stride = tensor->get_stride();
+ if (dim.size() == 1) {
+ int64_t const n = dim[0];
+ int64_t const leading_stride = (stride.size() == 1) ? stride[0] : 1;
+ tensor->set_dim({n, 1, 1, 1}).set_stride({leading_stride, 1, 1, 1});
+ }
+}
+
+namespace attn::score_modifiers {
+
+// clang-format off
+inline float get_negative_inf_value();
+
+inline std::shared_ptr<Tensor_attributes> causal_mask(
+ std::shared_ptr<Graph> graph,
+ std::shared_ptr<Tensor_attributes> attention_score
+);
+
+inline std::shared_ptr<Tensor_attributes> bias(
+ std::shared_ptr<Graph> graph,
+ std::shared_ptr<Tensor_attributes> attention_score,
+ std::shared_ptr<Tensor_attributes> bias_tensor
+);
+
+inline std::shared_ptr<Tensor_attributes> causal_mask_bottom_right(
+ std::shared_ptr<Graph> graph,
+ std::shared_ptr<Tensor_attributes> attention_score,
+ std::shared_ptr<Tensor_attributes> seq_len_q,
+ std::shared_ptr<Tensor_attributes> seq_len_kv
+);
+
+inline std::shared_ptr<Tensor_attributes> padding_mask(
+ std::shared_ptr<Graph> graph,
+ std::shared_ptr<Tensor_attributes> attention_score,
+ std::shared_ptr<Tensor_attributes> seq_len_kv,
+ std::shared_ptr<Tensor_attributes> seq_len_q
+);
+
+inline std::shared_ptr<Tensor_attributes> sliding_window_mask(
+ std::shared_ptr<Graph> graph,
+ std::shared_ptr<Tensor_attributes> attention_score,
+ DiagonalAlignment_t diagonal_alignment,
+ std::optional<int64_t> left_window,
+ std::optional<int64_t> right_window,
+ int64_t s_q,
+ int64_t s_kv,
+ std::shared_ptr<Tensor_attributes> s_q_ptr,
+ std::shared_ptr<Tensor_attributes> s_kv_ptr,
+ std::shared_ptr<Tensor_attributes> cu_s_q_ptr = nullptr,
+ std::shared_ptr<Tensor_attributes> cu_s_kv_ptr = nullptr
+);
+
+inline std::shared_ptr<Tensor_attributes> alibi_mask(
+ std::shared_ptr<Graph> graph,
+ std::shared_ptr<Tensor_attributes> attention_score,
+ std::shared_ptr<Tensor_attributes>& alibi_slopes,
+ int64_t h_q,
+ int64_t& alibi_slopes_size
+);
+
+inline error_t build_operation_subgraph(std::shared_ptr<Graph> graph);
+// clang-format on
+
+} // namespace attn::score_modifiers
+
+template <typename DerivedT>
+class SDPANodeBase : public NodeCRTP<DerivedT> {
+ protected:
+ using input_names = SDPA_attributes::input_names;
+ using output_names = SDPA_attributes::output_names;
+
+ std::shared_ptr<Tensor_attributes> rng_output;
+ std::shared_ptr<Tensor_attributes> alibi_slopes;
+ int64_t alibi_slopes_size = 0;
+
+ // Promote any 1-D seq_len / cu_seq_len / ragged-offset index tensors to the 4-D
+ // [n, 1, 1, 1] form the cuDNN backend requires (see promote_1d_index_tensor_to_4d).
+ void
+ promote_index_tensors_to_4d() {
+ for (auto& key :
+ {input_names::SEQ_LEN_Q, input_names::SEQ_LEN_KV, input_names::CU_SEQ_LEN_Q, input_names::CU_SEQ_LEN_KV}) {
+ auto const it = attributes.inputs.find(key);
+ if (it != attributes.inputs.end()) {
+ promote_1d_index_tensor_to_4d(it->second);
+ }
+ }
+ for (auto& [key, value] : attributes.inputs) {
+ CUDNN_FRONTEND_UNUSED(key);
+ if (value != nullptr) {
+ promote_1d_index_tensor_to_4d(value->get_ragged_offset());
+ }
+ }
+ for (auto& [key, value] : attributes.outputs) {
+ CUDNN_FRONTEND_UNUSED(key);
+ if (value != nullptr) {
+ promote_1d_index_tensor_to_4d(value->get_ragged_offset());
+ }
+ }
+ }
+
+ public:
+ SDPA_attributes attributes;
+
+ SDPANodeBase(SDPA_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP<DerivedT>(context), attributes(std::move(attributes_)) {}
+
+ bool
+ is_paged_v() const {
+ auto page_table_v_it = attributes.inputs.find(input_names::Page_table_V);
+ return ((page_table_v_it) != attributes.inputs.end() && page_table_v_it->second != nullptr);
+ }
+
+ bool
+ is_paged_k() const {
+ auto page_table_k_it = attributes.inputs.find(input_names::Page_table_K);
+ return ((page_table_k_it) != attributes.inputs.end() && page_table_k_it->second != nullptr);
+ }
+
+ bool
+ has_seq_len_q() const {
+ auto seq_len_Q_it = attributes.inputs.find(SDPA_attributes::input_names::SEQ_LEN_Q);
+ return ((seq_len_Q_it) != attributes.inputs.end() && seq_len_Q_it->second != nullptr);
+ }
+
+ bool
+ has_seq_len_kv() const {
+ auto seq_len_KV_it = attributes.inputs.find(SDPA_attributes::input_names::SEQ_LEN_KV);
+ return ((seq_len_KV_it) != attributes.inputs.end() && seq_len_KV_it->second != nullptr);
+ }
+
+ bool
+ has_cu_seq_len_q() const {
+ auto cu_seq_len_Q_it = attributes.inputs.find(SDPA_attributes::input_names::CU_SEQ_LEN_Q);
+ return ((cu_seq_len_Q_it) != attributes.inputs.end() && cu_seq_len_Q_it->second != nullptr);
+ }
+
+ bool
+ has_cu_seq_len_kv() const {
+ auto cu_seq_len_KV_it = attributes.inputs.find(SDPA_attributes::input_names::CU_SEQ_LEN_KV);
+ return ((cu_seq_len_KV_it) != attributes.inputs.end() && cu_seq_len_KV_it->second != nullptr);
+ }
+
+ // Helper function to detect MXFP8 (microscaling FP8) mode
+ // MXFP8 uses block-wise scale factors with E8M0 data type and F8_128x4 reordering
+ // When detected, we use block_scale_dequantize before matmuls instead of pointwise descale after
+ bool
+ is_mxfp8_scaling() const {
+ auto descale_q_it = attributes.inputs.find(input_names::Descale_Q);
+ if (descale_q_it == attributes.inputs.end() || descale_q_it->second == nullptr) {
+ return false;
+ }
+ auto const& descale_q = descale_q_it->second;
+ return (descale_q->get_data_type() == DataType_t::FP8_E8M0 &&
+ descale_q->get_reordering_type() == TensorReordering_t::F8_128x4);
+ }
+
+ // Helper function to infer KV sequence length
+ // Note that it cannot be run as part of infer_properties_node as
+ // this is being used in pre_validate_node
+ int64_t
+ infer_s_kv() const {
+ int64_t s_kv = -1;
+
+ auto get_input_dim = [this](const SDPA_attributes::input_names& input_name) {
+ auto const input_it = attributes.inputs.find(input_name);
+ if (input_it != attributes.inputs.end()) {
+ return input_it->second->get_dim();
+ } else {
+ return std::vector<int64_t>({-1, -1, -1, -1});
+ }
+ };
+
+ auto const& k_dim = get_input_dim(input_names::K);
+ auto const& v_dim = get_input_dim(input_names::V);
+
+ // If s_kv was set explicitly, use that
+ if (attributes.max_seq_len_kv.has_value()) {
+ s_kv = attributes.max_seq_len_kv.value();
+ }
+ // When one of K or V cache are paged, s_kv can be extracted directly
+ else if (!is_paged_k()) {
+ s_kv = k_dim[2];
+
+ } else if (!is_paged_v()) {
+ s_kv = v_dim[2];
+ } else {
+ CUDNN_FE_LOG_LABEL_ENDL(
+ "WARNING: maximum kv sequence length is being inferred. To set it explicitly, please use "
+ "\"set_paged_attention_max_seq_len_kv\"");
+
+ auto bias_it = attributes.inputs.find(input_names::Bias);
+ auto rng_it = attributes.outputs.find(output_names::RNG_DUMP);
+
+ // If there is a bias, extract it from there
+ if (bias_it != attributes.inputs.end() && bias_it->second != nullptr) {
+ s_kv = get_input_dim(input_names::Bias)[3];
+ // If there is an rng_dump output, extract it from there
+ } else if (rng_it != attributes.outputs.end() && rng_it->second != nullptr) {
+ s_kv = rng_it->second->get_dim()[3];
+ // When both caches are paged, and the above failed, we need to infer s_kv from the page table and
+ // container
+ } else {
+ // [b, 1, ceil(s_kv/block_size), 1]
+ auto page_table_dim_k = get_input_dim(input_names::Page_table_K);
+ // [b, h_k, block_size, d_k]
+ auto const container_dim_k = get_input_dim(input_names::K);
+ int64_t s_k = page_table_dim_k[2] * container_dim_k[2];
+
+ // [b, 1, ceil(s_kv/block_size), 1]
+ auto page_table_dim_v = get_input_dim(input_names::Page_table_V);
+ // [b, h_v, block_size, d_v]
+ auto const container_dim_v = get_input_dim(input_names::V);
+ int64_t s_v = page_table_dim_v[2] * container_dim_v[2];
+
+ s_kv = std::min(s_k, s_v);
+ }
+ }
+
+ return s_kv;
+ }
+
+ error_t
+ pre_validate_node() const override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating SDPANode " << attributes.name);
+
+ // check that Q, K, V, O tensors has been assigned
+ // check that dim and strides has been assigned and last stride is 1
+#define CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(port, port_map) \
+ { \
+ std::shared_ptr<Tensor_attributes> tensor_ptr = port_map.at(port); \
+ RETURN_CUDNN_FRONTEND_ERROR_IF(tensor_ptr->get_dim().size() != 4, \
+ error_code_t::ATTRIBUTE_NOT_SET, \
+ "The dim for " + std::string(#port) + " is invalid"); \
+ RETURN_CUDNN_FRONTEND_ERROR_IF(tensor_ptr->get_stride().size() != 4, \
+ error_code_t::ATTRIBUTE_NOT_SET, \
+ "The stride for " + std::string(#port) + " is invalid"); \
+ RETURN_CUDNN_FRONTEND_ERROR_IF( \
+ tensor_ptr->get_stride()[3] != 1, \
+ error_code_t::GRAPH_NOT_SUPPORTED, \
+ "The stride for the last dimension corresponding to the embedding size per head should be 1 for " + \
+ std::string(#port)); \
+ }
+
+ CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::Q, attributes.inputs);
+ CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::K, attributes.inputs);
+ CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::V, attributes.inputs);
+ CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(output_names::O, attributes.outputs);
+
+ if (attributes.generate_stats.value_or(false) == true) {
+ CUDNN_FE_VALIDATE_OUTPUT_TENSOR(output_names::Stats);
+ }
+
+ // If max is requested, validate that the output tensor is present
+ if (attributes.outputs.find(output_names::Max) != attributes.outputs.end() &&
+ attributes.outputs.at(output_names::Max) != nullptr) {
+ CUDNN_FE_VALIDATE_OUTPUT_TENSOR(output_names::Max);
+ }
+
+ // If sum_exp is requested, validate that the output tensor is present
+ if (attributes.outputs.find(output_names::Sum_exp) != attributes.outputs.end() &&
+ attributes.outputs.at(output_names::Sum_exp) != nullptr) {
+ CUDNN_FE_VALIDATE_OUTPUT_TENSOR(output_names::Sum_exp);
+ }
+
+#undef CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE
+
+ // validate backend limitations for the operation
+ auto validation_result =
+ attributes.validate_sdpa_support_surface(this->context, infer_s_kv(), is_paged_k(), is_paged_v());
+ if (validation_result.is_good() == false) {
+ return validation_result;
+ }
+
+ // return NOT_SET if sink_token present with 9.12 and below
+ RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 91300 &&
+ attributes.inputs.find(input_names::SINK_TOKEN) != attributes.inputs.end(),
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "SDPA with sink_token is not supported before 9.13.");
+
+ // Validate MXFP8 scale factors if present
+ if (is_mxfp8_scaling()) {
+ // MXFP8 requires cuDNN 9.21.0 or later
+ RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 92100,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "MXFP8 SDPA requires cuDNN 9.21.0 or later");
+
+ // Get device SM version from context
+ CHECK_CUDNN_FRONTEND_ERROR(this->context.populate_sm_version_from_device());
+ int32_t const sm_version = this->context.get_sm_version();
+ int32_t const prop_major = sm_version / 10;
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(10 != prop_major,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "MXFP8 SDPA is only supported on Blackwell Data Center architectures.");
+
+ auto const& q_dim = attributes.inputs.at(input_names::Q)->get_dim();
+ auto const& k_dim = attributes.inputs.at(input_names::K)->get_dim();
+ auto const& v_dim = attributes.inputs.at(input_names::V)->get_dim();
+ int64_t const b = q_dim[0];
+ int64_t const h_q = q_dim[1];
+ int64_t const s_q = q_dim[2];
+ int64_t const d = q_dim[3];
+ int64_t const h_k = k_dim[1];
+ int64_t const h_v = v_dim[1];
+ int64_t const s_kv = infer_s_kv();
+
+ // MXFP8 block size is fixed at 32
+ constexpr int64_t block_size = 32;
+ int64_t const d_scale = (d + block_size - 1) / block_size;
+ int64_t const s_scale = (s_kv + block_size - 1) / block_size;
+
+ // Validate Descale_Q
+ auto const& descale_q = attributes.inputs.at(input_names::Descale_Q);
+ RETURN_CUDNN_FRONTEND_ERROR_IF(descale_q->get_data_type() != DataType_t::FP8_E8M0,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA requires Descale_Q to have FP8_E8M0 data type");
+ RETURN_CUDNN_FRONTEND_ERROR_IF(descale_q->get_reordering_type() != TensorReordering_t::F8_128x4,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA requires Descale_Q to have F8_128x4 reordering");
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ descale_q->get_stride()[3] != 1,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "MXFP8 SDPA requires Descale_Q to have contiguous d_scale dimension (stride[3] == 1)");
+
+ // Validate Descale_K
+ auto const& descale_k = attributes.inputs.at(input_names::Descale_K);
+ RETURN_CUDNN_FRONTEND_ERROR_IF(descale_k->get_data_type() != DataType_t::FP8_E8M0,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA requires Descale_K to have FP8_E8M0 data type");
+ RETURN_CUDNN_FRONTEND_ERROR_IF(descale_k->get_reordering_type() != TensorReordering_t::F8_128x4,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA requires Descale_K to have F8_128x4 reordering");
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ descale_k->get_stride()[3] != 1,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "MXFP8 SDPA requires Descale_K to have contiguous d_scale dimension (stride[3] == 1)");
+
+ // Validate Descale_V
+ auto const& descale_v = attributes.inputs.at(input_names::Descale_V);
+ RETURN_CUDNN_FRONTEND_ERROR_IF(descale_v->get_data_type() != DataType_t::FP8_E8M0,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA requires Descale_V to have FP8_E8M0 data type");
+ RETURN_CUDNN_FRONTEND_ERROR_IF(descale_v->get_reordering_type() != TensorReordering_t::F8_128x4,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA requires Descale_V to have F8_128x4 reordering");
+ // // SF_V scales along s dimension (not d), so s_scale must be contiguous
+ // RETURN_CUDNN_FRONTEND_ERROR_IF(
+ // descale_v->get_stride()[2] != 1,
+ // error_code_t::GRAPH_NOT_SUPPORTED,
+ // "MXFP8 SDPA requires Descale_V to have contiguous s_scale dimension (stride[2] == 1)");
+
+ // Validate dimension consistency for SF_Q: [b, h_q, s_q_padded, d_scale_padded]
+ auto const& sf_q_dim = descale_q->get_dim();
+ RETURN_CUDNN_FRONTEND_ERROR_IF(sf_q_dim[0] != b || sf_q_dim[1] != h_q,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA: Descale_Q batch/head dimensions must match Q");
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ sf_q_dim[3] < d_scale,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA: Descale_Q d_scale dimension too small (expected >= " + std::to_string(d_scale) + ")");
+
+ // Validate dimension consistency for SF_K: [b, h_k, s_kv_padded, d_scale_padded]
+ auto const& sf_k_dim = descale_k->get_dim();
+ RETURN_CUDNN_FRONTEND_ERROR_IF(sf_k_dim[0] != b || sf_k_dim[1] != h_k,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA: Descale_K batch/head dimensions must match K");
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ sf_k_dim[3] < d_scale,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA: Descale_K d_scale dimension too small (expected >= " + std::to_string(d_scale) + ")");
+
+ // Validate dimension consistency for SF_V: [b, h_v, s_scale_padded, d_padded]
+ auto const& sf_v_dim = descale_v->get_dim();
+ RETURN_CUDNN_FRONTEND_ERROR_IF(sf_v_dim[0] != b || sf_v_dim[1] != h_v,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA: Descale_V batch/head dimensions must match V");
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ sf_v_dim[2] < s_scale,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA: Descale_V s_scale dimension too small (expected >= " + std::to_string(s_scale) + ")");
+
+ // Log MXFP8 configuration for debugging
+ getLogger() << "[cudnn_frontend] INFO: MXFP8 SDPA configuration validated:" << std::endl
+ << " Q dims: [" << b << ", " << h_q << ", " << s_q << ", " << d << "]" << std::endl
+ << " SF_Q dims: [" << sf_q_dim[0] << ", " << sf_q_dim[1] << ", " << sf_q_dim[2] << ", "
+ << sf_q_dim[3] << "]" << std::endl
+ << " SF_K dims: [" << sf_k_dim[0] << ", " << sf_k_dim[1] << ", " << sf_k_dim[2] << ", "
+ << sf_k_dim[3] << "]" << std::endl
+ << " SF_V dims: [" << sf_v_dim[0] << ", " << sf_v_dim[1] << ", " << sf_v_dim[2] << ", "
+ << sf_v_dim[3] << "]" << std::endl
+ << " Expected d_scale: " << d_scale << ", s_scale: " << s_scale << std::endl;
+ }
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ infer_properties_node() override final {
+ // If a 1-D length/offset index tensor [n] is supplied, promote it to the 4-D
+ // [n, 1, 1, 1] (contiguous) form the cuDNN backend requires.
+ promote_index_tensors_to_4d();
+
+ if (attributes.generate_stats.value_or(false)) {
+ auto stats = attributes.outputs.at(output_names::Stats);
+ auto stats_dim = stats->get_dim();
+
+ if (stats_dim.empty()) {
+ // Fill properties of virtual tensors
+ auto const& p_dim = attributes.inputs[input_names::Q]->get_dim();
+ auto b = p_dim[0];
+ auto h = p_dim[1];
+ auto s_q = p_dim[2];
+ stats->set_dim({b, h, s_q, 1}).set_stride({h * s_q, s_q, 1, 1});
+ }
+ }
+
+ if (attributes.outputs[output_names::Max] != nullptr) {
+ auto max = attributes.outputs.at(output_names::Max);
+
+ if (max->get_dim().empty()) {
+ // Fill properties of virtual tensors
+ auto const& p_dim = attributes.inputs[input_names::Q]->get_dim();
+ auto b = p_dim[0];
+ auto h = p_dim[1];
+ auto s_q = p_dim[2];
+ max->set_dim({b, h, s_q, 1}).set_stride({h * s_q, s_q, 1, 1});
+ }
+ }
+
+ if (attributes.outputs[output_names::Sum_exp] != nullptr) {
+ auto sum_exp = attributes.outputs.at(output_names::Sum_exp);
+
+ if (sum_exp->get_dim().empty()) {
+ // Fill properties of virtual tensors
+ auto const& p_dim = attributes.inputs[input_names::Q]->get_dim();
+ auto b = p_dim[0];
+ auto h = p_dim[1];
+ auto s_q = p_dim[2];
+ sum_exp->set_dim({b, h, s_q, 1}).set_stride({h * s_q, s_q, 1, 1});
+ }
+ }
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ post_validate_node() const override final {
+#define CUDNN_FE_VALIDATE_STRIDE(port, port_map) \
+ { \
+ auto const& t = port_map.find(port); \
+ RETURN_CUDNN_FRONTEND_ERROR_IF( \
+ t->second->get_stride().back() != 1, \
+ error_code_t::GRAPH_NOT_SUPPORTED, \
+ "The stride for the last dimension corresponding to the embedding size per head should be 1 for " + \
+ std::string(#port)); \
+ }
+
+ CUDNN_FE_VALIDATE_STRIDE(output_names::O, attributes.outputs);
+
+#undef CUDNN_FE_VALIDATE_STRIDE
+
+ return {error_code_t::OK, ""};
+ }
+
+ virtual int64_t
+ get_fe_workspace_size_node() const override final {
+ int64_t size = 0;
+
+ // align alibi slopes memory to 16 bytes
+ size += ((alibi_slopes_size + 15) / 16 * 16);
+
+ return size;
+ }
+
+ virtual error_t
+ collect_tensors_in_workspace_node(
+ std::unordered_map<Tensor_attributes::uid_t, std::tuple<int64_t, int64_t, std::vector<float>>>&
+ workspace_modifications,
+ int64_t& offset) const override final {
+ if (attributes.alibi_mask) {
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(Q, input_names::Q);
+ int64_t const h_q = Q->second->get_dim()[1];
+ auto alibi_slopes_vec = detail::get_alibi_slope(h_q);
+ workspace_modifications.emplace(alibi_slopes->get_uid(), std::make_tuple(0, offset, alibi_slopes_vec));
+ int64_t alibi_slopes_size_padded = ((alibi_slopes_size + 15) / 16 * 16);
+ offset = offset + alibi_slopes_size_padded;
+ }
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ collect_tensors_to_dump_node(
+ std::vector<std::pair<std::shared_ptr<Tensor_attributes>, char>>& tensors_to_dump) const override final {
+ std::unordered_set<Tensor_attributes::uid_t> seen_uids;
+ auto add_tensor = [&tensors_to_dump, &seen_uids](std::shared_ptr<Tensor_attributes> const& tensor) {
+ if (tensor == nullptr) {
+ return;
+ }
+ if (seen_uids.insert(tensor->get_uid()).second) {
+ tensors_to_dump.emplace_back(tensor, 'd');
+ }
+ };
+
+ auto const seq_len_q_it = attributes.inputs.find(input_names::SEQ_LEN_Q);
+ if (seq_len_q_it != attributes.inputs.end()) {
+ add_tensor(seq_len_q_it->second);
+ }
+
+ auto const seq_len_kv_it = attributes.inputs.find(input_names::SEQ_LEN_KV);
+ if (seq_len_kv_it != attributes.inputs.end()) {
+ add_tensor(seq_len_kv_it->second);
+ }
+
+ auto const cu_seq_len_q_it = attributes.inputs.find(input_names::CU_SEQ_LEN_Q);
+ if (cu_seq_len_q_it != attributes.inputs.end()) {
+ add_tensor(cu_seq_len_q_it->second);
+ }
+
+ auto const cu_seq_len_kv_it = attributes.inputs.find(input_names::CU_SEQ_LEN_KV);
+ if (cu_seq_len_kv_it != attributes.inputs.end()) {
+ add_tensor(cu_seq_len_kv_it->second);
+ }
+
+ for (auto const& tensor : {attributes.inputs.at(input_names::Q),
+ attributes.inputs.at(input_names::K),
+ attributes.inputs.at(input_names::V),
+ attributes.outputs.at(output_names::O)}) {
+ if (tensor != nullptr) {
+ add_tensor(tensor->get_ragged_offset());
+ }
+ }
+
+ auto const stats_it = attributes.outputs.find(output_names::Stats);
+ if (stats_it != attributes.outputs.end() && stats_it->second != nullptr) {
+ add_tensor(stats_it->second->get_ragged_offset());
+ }
+
+ auto const max_it = attributes.outputs.find(output_names::Max);
+ if (max_it != attributes.outputs.end() && max_it->second != nullptr) {
+ add_tensor(max_it->second->get_ragged_offset());
+ }
+
+ auto const sum_exp_it = attributes.outputs.find(output_names::Sum_exp);
+ if (sum_exp_it != attributes.outputs.end() && sum_exp_it->second != nullptr) {
+ add_tensor(sum_exp_it->second->get_ragged_offset());
+ }
+
+ return {error_code_t::OK, ""};
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ j["is_mxfp8"] = is_mxfp8_scaling();
+ j["unfuse_fma"] = attributes.unfuse_fma;
+ if (auto const rescale_threshold = get_rescale_threshold_from_env(); rescale_threshold.has_value()) {
+ j["rescale_threshold"] = rescale_threshold.value();
+ }
+ if (is_mxfp8_scaling()) {
+ j.update(R"({"tag": "SDPA_MXFP8_FWD"})"_json);
+ } else if (attributes.mma_core_mode == DataType_t::FP8_E4M3 ||
+ attributes.mma_core_mode == DataType_t::FP8_E5M2) {
+ j.update(R"({"tag": "SDPA_FP8_FWD"})"_json);
+ } else {
+ j.update(R"({"tag": "SDPA"})"_json);
+ }
+ }
+#endif
+};
+
+class CompositeSDPANode : public SDPANodeBase<CompositeSDPANode> {
+ public:
+ CompositeSDPANode(SDPA_attributes&& attributes_, detail::Context const& context)
+ : SDPANodeBase(std::move(attributes_), context) {}
+
+ Type
+ getType() override final {
+ return Type::COMPOSITE;
+ }
+
+ error_t
+ expand_node() override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for CompositeSDPANode node " << attributes.name);
+
+ // DO NOT REMOVE
+ // input data type is needed for:
+ // - aType of bmm2
+ // - dropout scale in pre 8.9.3
+ attributes.fill_from_context(this->context);
+
+ // Gather dim to fill properties of virtual tensors
+ auto const& q_dim = attributes.inputs[input_names::Q]->get_dim();
+ auto b = q_dim[0];
+ auto h_q = q_dim[1];
+ auto s_q = q_dim[2];
+ auto d_qk = q_dim[3];
+ auto const& k_dim = attributes.inputs[input_names::K]->get_dim();
+ auto h_k = k_dim[1];
+ auto const& v_dim = attributes.inputs[input_names::V]->get_dim();
+ auto h_v = v_dim[1];
+ auto d_v = v_dim[3];
+ // Infer s_kv
+ int64_t s_kv = infer_s_kv();
+
+ // Check if using MXFP8 (microscaling FP8) with block-wise scale factors
+ // Need to check this early because MXFP8 dequantization must happen before K transpose
+ bool const use_mxfp8 = is_mxfp8_scaling();
+
+ std::shared_ptr<Tensor_attributes> k_cache;
+ if (!is_paged_k()) {
+ if (use_mxfp8) {
+ // MXFP8: Transpose K and SF_K first, then dequantize
+ // The backend expects both K and its scale factor in transposed layout
+
+ // Step 1: Transpose K -> KT by swapping dims and strides
+ // K = {b, h_k, s_kv, d_qk} -> KT = {b, h_k, d_qk, s_kv}
+ std::vector<int64_t> kt_dim = attributes.inputs[input_names::K]->get_dim();
+ std::vector<int64_t> kt_stride = attributes.inputs[input_names::K]->get_stride();
+ std::swap(kt_dim[2], kt_dim[3]);
+ std::swap(kt_stride[2], kt_stride[3]);
+ attributes.inputs[input_names::K]->set_dim(kt_dim);
+ attributes.inputs[input_names::K]->set_stride(kt_stride);
+
+ // Step 2: Transpose SF_K similarly
+ // SF_K = {b, h_k, s_kv_scale, d_qk_scale} -> {b, h_k, d_qk_scale, s_kv_scale}
+ auto& sf_k = attributes.inputs[input_names::Descale_K];
+ std::vector<int64_t> sf_k_dim = sf_k->get_dim();
+ std::vector<int64_t> sf_k_stride = sf_k->get_stride();
+ std::swap(sf_k_dim[2], sf_k_dim[3]);
+ std::swap(sf_k_stride[2], sf_k_stride[3]);
+ sf_k->set_dim(sf_k_dim);
+ sf_k->set_stride(sf_k_stride);
+
+ // Step 3: Dequantize transposed K with transposed SF_K
+ auto dequant_k_attrs = Block_scale_dequantize_attributes()
+ .set_name("dequant_k")
+ .set_block_size({1, 32}); // Standard MXFP8 block size
+
+ auto k_dequant = std::make_shared<Tensor_attributes>();
+ k_dequant->set_is_virtual(true);
+ k_dequant->set_dim(kt_dim);
+ k_dequant->set_stride(kt_stride);
+
+ block_scale_dequantize(attributes.inputs[input_names::K], sf_k, dequant_k_attrs, k_dequant);
+
+ k_cache = k_dequant;
+ } else {
+ // Non-MXFP8: map K->KT directly
+ // cuDNN frontend API attention requires Q, K, V where
+ // Q = {b, h_q, s_q, d_qk}
+ // K = {b, h_k, s_kv, d_qk}
+ // V = {b, h_v, s_kv, d_v}
+ // but cuDNN backend API attention requires Q, KT, V
+ // Q = {b, h_q, s_q, d_qk}
+ // KT = {b, h_k, d_qk, s_kv}
+ // V = {b, h_v, s_kv, d_v}
+ // So the code below maps the K->KT
+ std::vector<int64_t> temp_vec;
+
+ temp_vec = attributes.inputs[input_names::K]->get_dim();
+ std::swap(temp_vec[2], temp_vec[3]);
+ attributes.inputs[input_names::K]->set_dim(temp_vec);
+
+ temp_vec = attributes.inputs[input_names::K]->get_stride();
+ std::swap(temp_vec[2], temp_vec[3]);
+ attributes.inputs[input_names::K]->set_stride(temp_vec);
+
+ k_cache = attributes.inputs[input_names::K];
+ }
+ } else {
+ // Create a paged cache load operation
+ auto paged_cache_load_attributes_k = PagedCacheLoad_attributes().set_name("paged_k_cache_operation");
+ // Need to create virtual tensor descriptor for yOut here as it cannot be inferred
+ // K-cache has BHDS layout
+ k_cache = std::make_shared<Tensor_attributes>();
+ k_cache->set_is_virtual(true);
+ k_cache->set_dim({b, h_k, d_qk, s_kv});
+ k_cache->set_stride({d_qk * s_kv * h_k, d_qk * s_kv, 1, d_qk});
+ k_cache->set_data_type(attributes.inputs[input_names::K]->get_data_type());
+ paged_cache_load(attributes.inputs[input_names::K],
+ attributes.inputs[input_names::SEQ_LEN_KV],
+ attributes.inputs[input_names::Page_table_K],
+ paged_cache_load_attributes_k,
+ k_cache);
+ }
+
+ // This tensor tracks the main chain of data flow
+ std::shared_ptr<Tensor_attributes> last_output;
+
+ // Prepare Q and K for first matmul
+ // For MXFP8: Q is dequantized here, K was dequantized above before transpose
+ // For regular FP8: use raw tensors, descale applied after matmul
+ std::shared_ptr<Tensor_attributes> q_for_bmm1;
+ std::shared_ptr<Tensor_attributes> k_for_bmm1;
+
+ if (use_mxfp8) {
+ // MXFP8: Dequantize Q using block scale factors
+ auto dequant_q_attrs = Block_scale_dequantize_attributes()
+ .set_name("dequant_q")
+ .set_block_size({1, 32}); // Standard MXFP8 block size
+ auto q_dequant = std::make_shared<Tensor_attributes>();
+ q_dequant->set_is_virtual(true);
+ q_dequant->set_dim(attributes.inputs[input_names::Q]->get_dim());
+ q_dequant->set_stride(attributes.inputs[input_names::Q]->get_stride());
+ block_scale_dequantize(attributes.inputs[input_names::Q],
+ attributes.inputs[input_names::Descale_Q],
+ dequant_q_attrs,
+ q_dequant);
+ q_for_bmm1 = q_dequant;
+
+ // K was already dequantized and transposed above
+ k_for_bmm1 = k_cache;
+ } else {
+ q_for_bmm1 = attributes.inputs[input_names::Q];
+ k_for_bmm1 = k_cache;
+ }
+
+ //// Q * K
+ auto bmm1_attributes = Matmul_attributes()
+ .set_name("bmm1")
+ .set_m_override(attributes.inputs[input_names::SEQ_LEN_Q])
+ .set_n_override(attributes.inputs[input_names::SEQ_LEN_KV]);
+
+ if (attributes.padding_mask) {
+ bmm1_attributes.set_padding(0.0);
+ }
+
+ auto const& bmm1_output = matmul(q_for_bmm1, k_for_bmm1, bmm1_attributes);
+ // Setting dim and strides as pointwise op wont have knowledge of how to do it for mha.
+ bmm1_output->set_dim({b, h_q, s_q, s_kv}).set_stride({h_q * s_q * s_kv, s_q * s_kv, s_kv, 1});
+ last_output = bmm1_output;
+
+ //// Optional Attn scale
+ // In case user provided a scalar value, do a fused scalar.
+ if (attributes.attn_scale_value.has_value()) {
+ attributes.inputs[input_names::Attn_scale] =
+ std::make_shared<Tensor_attributes>(attributes.attn_scale_value.value());
+ }
+
+ // If attn scale present, add a pointwise mul node
+ if (attributes.inputs[input_names::Attn_scale]) {
+ Pointwise_attributes scale_attributes;
+ scale_attributes.set_name("attn_scale").set_mode(PointwiseMode_t::MUL);
+ auto const& attn_scale_output =
+ pointwise(last_output, attributes.inputs[input_names::Attn_scale], scale_attributes);
+ last_output = attn_scale_output;
+ }
+
+ // Descale Q (only for non-MXFP8 per-tensor scaling)
+ // For MXFP8, descaling was done via block_scale_dequantize before matmul
+ if (!use_mxfp8 && attributes.inputs.find(input_names::Descale_Q) != attributes.inputs.end() &&
+ attributes.inputs.at(input_names::Descale_Q) != nullptr) {
+ auto descale_q_attributes = Pointwise_attributes().set_mode(PointwiseMode_t::MUL).set_name("descale_q");
+ last_output = pointwise(last_output, attributes.inputs.at(input_names::Descale_Q), descale_q_attributes);
+ }
+
+ // Descale K (only for non-MXFP8 per-tensor scaling)
+ // For MXFP8, descaling was done via block_scale_dequantize before matmul
+ if (!use_mxfp8 && attributes.inputs.find(input_names::Descale_K) != attributes.inputs.end() &&
+ attributes.inputs.at(input_names::Descale_K) != nullptr) {
+ auto descale_k_attributes = Pointwise_attributes().set_mode(PointwiseMode_t::MUL).set_name("descale_k");
+ last_output = pointwise(last_output, attributes.inputs.at(input_names::Descale_K), descale_k_attributes);
+ }
+
+ if (attributes.attention_score_modifier != nullptr) {
+ auto graph_ = std::make_shared<Graph>();
+ std::shared_ptr<INode> node_ = std::static_pointer_cast<INode>(graph_);
+ node_->context = this->context;
+ last_output = attributes.attention_score_modifier(graph_, last_output);
+ sub_nodes.emplace_back(node_);
+ }
+
+ // Optional bias
+ if (attributes.inputs.find(input_names::Bias) != attributes.inputs.end() &&
+ attributes.inputs[input_names::Bias]) {
+ auto graph_ = std::make_shared<Graph>();
+ std::shared_ptr<INode> node_ = std::static_pointer_cast<INode>(graph_);
+ node_->context = this->context;
+ last_output = attn::score_modifiers::bias(graph_, last_output, attributes.inputs[input_names::Bias]);
+ sub_nodes.emplace_back(node_);
+ }
+
+ if (attributes.alibi_mask) {
+ auto graph_ = std::make_shared<Graph>();
+ std::shared_ptr<INode> node_ = std::static_pointer_cast<INode>(graph_);
+ node_->context = this->context;
+ last_output = attn::score_modifiers::alibi_mask(graph_, last_output, alibi_slopes, h_q, alibi_slopes_size);
+ sub_nodes.emplace_back(node_);
+ }
+
+ // There are two cases of applying padding mask
+ // 1. when actual seq_len is less than or equal to max_seq_len
+ if (attributes.padding_mask) {
+ auto graph_ = std::make_shared<Graph>();
+ std::shared_ptr<INode> node_ = std::static_pointer_cast<INode>(graph_);
+ node_->context = this->context;
+ last_output = attn::score_modifiers::padding_mask(graph_,
+ last_output,
+ attributes.inputs[input_names::SEQ_LEN_KV],
+ attributes.inputs[input_names::SEQ_LEN_Q]);
+ sub_nodes.emplace_back(node_);
+ }
+
+ // 2. (bug in cudnn backend) no padding with max_seq_len%64!=0
+ if ((s_kv % 64 != 0) && (!(attributes.padding_mask)) && (detail::get_backend_version() < 90000)) {
+ auto col_index_attributes =
+ Pointwise_attributes().set_name("gen_col_index").set_mode(PointwiseMode_t::GEN_INDEX).set_axis(3);
+ auto col_index_output = pointwise(last_output, col_index_attributes);
+ // scalar seq_kv only needs to be passed in case there in no padding mask and seq_kv is not multiple of 64.
+ // Also future versions of cudnn will not need it, hence tensor is pre-fixed.
+ auto scalar_max_seq_kv = std::make_shared<Tensor_attributes>(static_cast<int32_t>(s_kv));
+
+ auto col_less_seq_kv_attributes =
+ Pointwise_attributes().set_name("col_less_seq_kv").set_mode(PointwiseMode_t::CMP_LT);
+ auto col_less_seq_kv_output = pointwise(col_index_output, scalar_max_seq_kv, col_less_seq_kv_attributes);
+
+ // Lower attributes to binary select attributes
+ auto negative_inf_padding =
+ std::make_shared<Tensor_attributes>(attn::score_modifiers::get_negative_inf_value());
+ auto binary_select_attributes =
+ Pointwise_attributes().set_name("binary_select").set_mode(PointwiseMode_t::BINARY_SELECT);
+ auto padding_mask_output =
+ pointwise(last_output, negative_inf_padding, col_less_seq_kv_output, binary_select_attributes);
+ last_output = padding_mask_output;
+ }
+
+ // Apply (bottom-right) causal masking (with right bound) and/or set the left bound
+ if (attributes.left_bound.has_value() || attributes.right_bound.has_value()) {
+ auto graph_ = std::make_shared<Graph>();
+ std::shared_ptr<INode> node_ = std::static_pointer_cast<INode>(graph_);
+ node_->context = this->context;
+
+ auto s_kv_ptr = attributes.inputs.find(input_names::SEQ_LEN_KV) != attributes.inputs.end()
+ ? attributes.inputs[input_names::SEQ_LEN_KV]
+ : nullptr;
+ auto s_q_ptr = attributes.inputs.find(input_names::SEQ_LEN_Q) != attributes.inputs.end()
+ ? attributes.inputs[input_names::SEQ_LEN_Q]
+ : nullptr;
+
+ last_output = attn::score_modifiers::sliding_window_mask(graph_,
+ last_output,
+ attributes.diagonal_alignment,
+ attributes.left_bound,
+ attributes.right_bound,
+ s_q,
+ s_kv,
+ s_q_ptr,
+ s_kv_ptr,
+ /*cu_s_q_ptr=*/nullptr,
+ /*cu_s_kv_ptr=*/nullptr);
+ sub_nodes.emplace_back(node_);
+ }
+
+ // Lower attributes to softmax attributes
+ auto softmax_output = std::make_shared<Tensor_attributes>();
+ softmax_output->set_is_virtual(true);
+
+ auto softmax_attributes = Softmax_attributes().set_name("softmax");
+ // Set sink for softmax if user has provided a sink tensor
+ if (attributes.inputs.find(input_names::SINK_TOKEN) != attributes.inputs.end()) {
+ softmax_attributes.set_sink(attributes.inputs[input_names::SINK_TOKEN]);
+ }
+ // Special non-functional-style call. Needed because output already created and provided to user.
+ softmax(last_output,
+ softmax_attributes,
+ softmax_output,
+ attributes.outputs[output_names::Stats],
+ attributes.outputs[output_names::Max],
+ attributes.outputs[output_names::Sum_exp]);
+ last_output = softmax_output;
+
+ // Two cases for training: dropout present or not
+ bool dropout_present = false;
+ auto const& dropout_mask = attributes.inputs.find(input_names::Dropout_mask);
+ bool const is_dropout_custom = (dropout_mask != attributes.inputs.end()) && (dropout_mask->second != nullptr);
+ if (attributes.dropout_probability.has_value()) {
+ dropout_present = true;
+ // Special case: Skip dropout when 0.0 probability. Only do for 8.9.3 and up as rng isn't optional earlier.
+ if (detail::get_backend_version() > 8902 && attributes.dropout_probability.value() == 0.0) {
+ dropout_present = false;
+ }
+ } else if (is_dropout_custom) {
+ dropout_present = true;
+ }
+
+ if (dropout_present) {
+ if (is_dropout_custom) {
+ auto dropout_scale_attributes =
+ Pointwise_attributes().set_name("dropout_scale_mul").set_mode(PointwiseMode_t::MUL);
+ auto const& dropout_scale_output =
+ pointwise(last_output, attributes.inputs[input_names::Dropout_scale], dropout_scale_attributes);
+
+ auto mask_attributes =
+ Pointwise_attributes().set_name("dropout_mask_mul").set_mode(PointwiseMode_t::MUL);
+ auto const& dropout_mask_output =
+ pointwise(dropout_scale_output, dropout_mask->second, mask_attributes);
+ last_output = dropout_mask_output;
+ } else {
+ if (attributes.outputs[output_names::RNG_DUMP] != nullptr) {
+ rng_output = attributes.outputs[output_names::RNG_DUMP];
+ rng(attributes.inputs[input_names::Seed],
+ attributes.inputs[input_names::Offset],
+ Rng_attributes()
+ .set_name("rng")
+ .set_distribution(RngDistribution_t::BERNOULLI)
+ .set_bernoulli_probability(1.0 - attributes.dropout_probability.value()),
+ rng_output);
+ } else {
+ rng_output = rng(attributes.inputs[input_names::Seed],
+ attributes.inputs[input_names::Offset],
+ Rng_attributes()
+ .set_name("rng")
+ .set_distribution(RngDistribution_t::BERNOULLI)
+ .set_bernoulli_probability(1.0 - attributes.dropout_probability.value()));
+ rng_output
+ // Hard coding dim and strides as rng output can no inputs to infer it from.
+ ->set_dim({b, h_q, s_q, s_kv})
+ .set_stride({h_q * s_q * s_kv, s_q * s_kv, s_kv, 1});
+ }
+
+ auto mask_attributes =
+ Pointwise_attributes().set_name("dropout_mask_mul").set_mode(PointwiseMode_t::MUL);
+ auto const& dropout_mask_output = pointwise(last_output, rng_output, mask_attributes);
+ last_output = dropout_mask_output;
+
+ std::shared_ptr<cudnn_frontend::graph::Tensor_attributes> dropout_scale = nullptr;
+
+ if (detail::get_backend_version() < 8903) {
+ half dropout_scale_value = __float2half(1.0f / (1.0f - attributes.dropout_probability.value()));
+ dropout_scale = std::make_shared<Tensor_attributes>(dropout_scale_value);
+ } else {
+ float dropout_scale_value = (1.0f / (1.0f - attributes.dropout_probability.value()));
+ dropout_scale = std::make_shared<Tensor_attributes>(dropout_scale_value);
+ }
+
+ auto dropout_scale_attributes =
+ Pointwise_attributes().set_name("dropout_scale").set_mode(PointwiseMode_t::MUL);
+ auto const& dropout_scale_output = pointwise(last_output, dropout_scale, dropout_scale_attributes);
+ last_output = dropout_scale_output;
+ }
+ }
+
+ // Amax S
+ if (attributes.outputs.find(output_names::Amax_S) != attributes.outputs.end() &&
+ attributes.outputs.at(output_names::Amax_S) != nullptr) {
+ auto amax_attributes = Reduction_attributes().set_name("amax_s").set_mode(ReductionMode_t::AMAX);
+ // Special non-functional-style call. Needed because output already created and provided to user.
+ reduction(last_output, amax_attributes, attributes.outputs.at(output_names::Amax_S));
+ }
+
+ // Scale S
+ if (attributes.inputs.find(input_names::Scale_S) != attributes.inputs.end() &&
+ attributes.inputs.at(input_names::Scale_S) != nullptr) {
+ auto scale_s_attributes = Pointwise_attributes().set_name("scale_s").set_mode(PointwiseMode_t::MUL);
+ last_output = pointwise(last_output, attributes.inputs.at(input_names::Scale_S), scale_s_attributes);
+ }
+
+ // Lower attributes to bmm2 attributes
+ // Requirement by cudnn backend to take in bmm2 aType as i/o type.
+ last_output->set_data_type(attributes.inputs[input_names::Q]->get_data_type());
+
+ auto const& seq_len_q = attributes.inputs[input_names::SEQ_LEN_Q];
+ auto const& seq_len_kv = attributes.inputs[input_names::SEQ_LEN_KV];
+ // auto const& V = attributes.inputs[input_names::V];
+ auto const& O = attributes.outputs[output_names::O];
+
+ std::shared_ptr<Tensor_attributes> v_cache;
+
+ if (!is_paged_v()) {
+ v_cache = attributes.inputs[input_names::V];
+ } else {
+ auto paged_cache_load_attributes_v = PagedCacheLoad_attributes().set_name("paged_v_cache_operation");
+ v_cache = std::make_shared<Tensor_attributes>();
+ v_cache->set_dim({b, h_v, s_kv, d_v})
+ .set_stride({d_v * s_kv * h_v, d_v * s_kv, d_v, 1})
+ .set_data_type(attributes.inputs[input_names::V]->get_data_type());
+ v_cache->set_is_virtual(true);
+ paged_cache_load(attributes.inputs[input_names::V],
+ attributes.inputs[input_names::SEQ_LEN_KV],
+ attributes.inputs[input_names::Page_table_V],
+ paged_cache_load_attributes_v,
+ v_cache);
+ }
+
+ //// S * V
+ if (attributes.mma_core_mode == DataType_t::HALF) {
+ auto bmm2_attributes =
+ Matmul_attributes().set_name("bmm2").set_m_override(seq_len_q).set_k_override(seq_len_kv);
+ // Special non-functional-style call. Needed because output already created and provided to user.
+ matmul(last_output, v_cache, bmm2_attributes, O);
+ } else if (attributes.mma_core_mode == DataType_t::FP8_E4M3 ||
+ attributes.mma_core_mode == DataType_t::FP8_E5M2) {
+ if (use_mxfp8) {
+ // MXFP8: Dequantize V using block scale factors
+ // SF_V should be provided with dims [b, h, s_scale, d] where s is scaled
+ // (different from SF_Q/SF_K which have d scaled)
+ // No transformation needed - use SF_V as provided
+
+ // Dequantize V with SF_V
+ auto dequant_v_attrs = Block_scale_dequantize_attributes()
+ .set_name("dequant_v")
+ .set_block_size({1, 32}); // Standard MXFP8 block size
+ auto v_dequant = std::make_shared<Tensor_attributes>();
+ v_dequant->set_is_virtual(true);
+ v_dequant->set_dim(v_cache->get_dim());
+ v_dequant->set_stride(v_cache->get_stride());
+ block_scale_dequantize(v_cache, attributes.inputs[input_names::Descale_V], dequant_v_attrs, v_dequant);
+
+ // Use regular matmul with dequantized inputs
+ auto bmm2_attributes =
+ Matmul_attributes().set_name("bmm2").set_m_override(seq_len_q).set_k_override(seq_len_kv);
+ // Special non-functional-style call. Needed because output already created and provided to user.
+ matmul(last_output, v_dequant, bmm2_attributes, O);
+
+ // Compute Amax_O for MXFP8
+ auto const& amax_o = attributes.outputs.at(output_names::Amax_O);
+ if (amax_o != nullptr) {
+ auto amax_attributes = Reduction_attributes().set_name("amax_o").set_mode(ReductionMode_t::AMAX);
+ // Special non-functional-style call. Needed because output already created and provided to user.
+ reduction(O, amax_attributes, amax_o);
+ }
+ } else {
+ // Regular per-tensor FP8 scaling
+ auto const& descale_s = attributes.inputs.at(input_names::Descale_S);
+ auto const& descale_v = attributes.inputs.at(input_names::Descale_V);
+ auto const& scale_o = attributes.inputs.at(input_names::Scale_O);
+ auto const& amax_o = attributes.outputs.at(output_names::Amax_O);
+
+ auto bmm2_attributes =
+ Matmul_fp8_attributes().set_name("bmm2").set_m_override(seq_len_q).set_k_override(seq_len_kv);
+ // Special non-functional-style call. Needed because output already created and provided to user.
+ matmul_fp8(last_output, v_cache, descale_s, descale_v, scale_o, bmm2_attributes, O, amax_o);
+ }
+ } else {
+ RETURN_CUDNN_FRONTEND_ERROR_IF(true, error_code_t::GRAPH_NOT_SUPPORTED, "Unsupported MMA core mode");
+ }
+
+ return {error_code_t::OK, ""};
+ }
+};
+
+class CompositeSDPABackwardNode : public NodeCRTP<CompositeSDPABackwardNode> {
+ using input_names = SDPA_backward_attributes::input_names;
+ using output_names = SDPA_backward_attributes::output_names;
+
+ private:
+ // non-virtual node gpu tensors
+ std::shared_ptr<Tensor_attributes> dQ_accum;
+ int64_t dQ_accum_size = 0;
+ std::shared_ptr<Tensor_attributes> dK_fullhead;
+ int64_t dK_fullhead_size = 0;
+ std::shared_ptr<Tensor_attributes> dV_fullhead;
+ int64_t dV_fullhead_size = 0;
+ std::shared_ptr<Tensor_attributes> softmax_sum;
+ int64_t softmax_sum_size = 0;
+ std::shared_ptr<Tensor_attributes> alibi_slopes;
+ int64_t alibi_slopes_size = 0;
+
+ mutable bool has_workaround_padding_mask = false; // Will be edited in pre_validate_node()
+ mutable int32_t s_q_for_workaround_padding_mask = 0; // Will be edited in pre_validate_node()
+ mutable int32_t s_kv_for_workaround_padding_mask = 0; // Will be edited in pre_validate_node()
+ mutable std::shared_ptr<Tensor_attributes>
+ workaround_padding_mask_seq_len_q; // Will be edited in pre_validate_node()
+ mutable std::shared_ptr<Tensor_attributes>
+ workaround_padding_mask_seq_len_kv; // Will be edited in pre_validate_node()
+ mutable int64_t batch_size_for_workaround_padding_mask = 0; // Will be edited in pre_validate_node()
+ mutable bool is_deterministic_algorithm_supported_on_blackwell = false; // Will be edited in pre_validate_node()
+ mutable bool is_d256_on_blackwell = false; // Will be edited in pre_validate_node()
+
+ // Promote any 1-D seq_len / ragged-offset index tensors to the 4-D
+ // [n, 1, 1, 1] form the cuDNN backend requires (see promote_1d_index_tensor_to_4d).
+ void
+ promote_index_tensors_to_4d() {
+ // TODO: Handle CU_SEQ_LEN_Q and CU_SEQ_LEN_KV once bprop supports these.
+ for (auto& key : {input_names::SEQ_LEN_Q, input_names::SEQ_LEN_KV}) {
+ auto const it = attributes.inputs.find(key);
+ if (it != attributes.inputs.end()) {
+ promote_1d_index_tensor_to_4d(it->second);
+ }
+ }
+ for (auto& [key, value] : attributes.inputs) {
+ CUDNN_FRONTEND_UNUSED(key);
+ if (value != nullptr) {
+ promote_1d_index_tensor_to_4d(value->get_ragged_offset());
+ }
+ }
+ for (auto& [key, value] : attributes.outputs) {
+ CUDNN_FRONTEND_UNUSED(key);
+ if (value != nullptr) {
+ promote_1d_index_tensor_to_4d(value->get_ragged_offset());
+ }
+ }
+ }
+
+ public:
+ mutable SDPA_backward_attributes attributes; // Will be edited in pre_validate_node() for workaround padding mask
+
+ CompositeSDPABackwardNode(SDPA_backward_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP(context), attributes(std::move(attributes_)) {}
+
+ Type
+ getType() override final {
+ return Type::COMPOSITE;
+ }
+
+ error_t
+ pre_validate_node() const override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating CompositeSDPABackwardNode" << attributes.name);
+
+ // check that Q, K, V, O, stats, dO, dQ, dK, dV tensors has been assigned
+ // check that dim and strides has been assigned and last stride is 1
+#define CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(port, port_map) \
+ { \
+ std::shared_ptr<Tensor_attributes> tensor_ptr = port_map.at(port); \
+ RETURN_CUDNN_FRONTEND_ERROR_IF(tensor_ptr->get_dim().size() != 4, \
+ error_code_t::ATTRIBUTE_NOT_SET, \
+ "The dim for " + std::string(#port) + " is invalid"); \
+ RETURN_CUDNN_FRONTEND_ERROR_IF(tensor_ptr->get_stride().size() != 4, \
+ error_code_t::ATTRIBUTE_NOT_SET, \
+ "The stride for " + std::string(#port) + " is invalid"); \
+ RETURN_CUDNN_FRONTEND_ERROR_IF( \
+ tensor_ptr->get_stride()[3] != 1, \
+ error_code_t::GRAPH_NOT_SUPPORTED, \
+ "The stride for the last dimension corresponding to the embedding size per head should be 1 for " + \
+ std::string(#port)); \
+ }
+
+ CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::Q, attributes.inputs);
+ CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::K, attributes.inputs);
+ CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::V, attributes.inputs);
+ CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::O, attributes.inputs);
+ CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::Stats, attributes.inputs);
+ CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::dO, attributes.inputs);
+ CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(output_names::dQ, attributes.outputs);
+ CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(output_names::dK, attributes.outputs);
+ CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(output_names::dV, attributes.outputs);
+
+#undef CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE
+
+ // validate backend limitations for the operation
+ // clang-format off
+ int64_t s_q = attributes.inputs.at(input_names::Q)->get_dim()[2];
+ int64_t s_kv = attributes.inputs.at(input_names::V)->get_dim()[2];
+ int64_t h_q = attributes.inputs.at(input_names::Q)->get_dim()[1];
+ int64_t h_k = attributes.inputs.at(input_names::K)->get_dim()[1];
+ int64_t h_v = attributes.inputs.at(input_names::V)->get_dim()[1];
+ int64_t d_qk = attributes.inputs.at(input_names::Q)->get_dim()[3];
+ int64_t d_v = attributes.inputs.at(input_names::V)->get_dim()[3];
+
+ bool const is_ragged = attributes.inputs.at(input_names::Q)->get_ragged_offset() ||
+ attributes.inputs.at(input_names::K)->get_ragged_offset() ||
+ attributes.inputs.at(input_names::V)->get_ragged_offset() ||
+ attributes.inputs.at(input_names::O)->get_ragged_offset();
+
+ auto const& bias_mask = attributes.inputs.find(input_names::Bias);
+ bool const is_bias = (bias_mask != attributes.inputs.end() && bias_mask->second != nullptr);
+ auto const& dbias_mask = attributes.outputs.find(output_names::dBias);
+ bool const is_dbias = (dbias_mask != attributes.outputs.end() && dbias_mask->second != nullptr);
+
+ auto const& dropout_mask = attributes.inputs.find(input_names::Dropout_mask);
+ bool const is_dropout_custom = (dropout_mask != attributes.inputs.end()) && (dropout_mask->second != nullptr);
+ bool const is_dropout = attributes.dropout_probability.has_value() || is_dropout_custom;
+
+ auto const& rng_tensor = attributes.outputs.find(output_names::RNG_DUMP);
+ bool const is_rng = (rng_tensor != attributes.outputs.end() && rng_tensor->second != nullptr);
+
+ // validation TODO:
+ // - validate stats has valid dims
+ // - validate Q and dQ have the same dims
+
+ // Stop s_q = S_kv = 1 from running
+ RETURN_CUDNN_FRONTEND_ERROR_IF(s_q == 1 && s_kv == 1,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "s_q = s_kv = 1 is not supported.");
+
+ // Bug workarounds for known problematic versions (TE constraint)
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ detail::get_backend_version() == 91000 || detail::get_backend_version() == 91001,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "SDPA FP16/BF16 backward is not supported on cuDNN 9.10.0/9.10.1 due to known bugs. "
+ "Please consider upgrading to 9.10.2 or newer.");
+
+ // 9.14.0 sliding window bug: non-causal + s_kv > 1024 + sliding window
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ detail::get_backend_version() == 91400 && s_kv > 1024 && attributes.left_bound.has_value() &&
+ !attributes.has_causal_like_masking(),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "cuDNN 9.14.0 has a known bug with non-causal + s_kv > 1024 + sliding window attention. "
+ "Please consider upgrading to 9.14.1 or newer.");
+
+ CHECK_CUDNN_FRONTEND_ERROR(context.populate_sm_version_from_device());
+ int32_t const sm_version = context.get_sm_version();
+ int32_t const prop_major = sm_version / 10;
+
+ if (prop_major == 9) {
+ // validate basic dimension requirements
+
+ if ((detail::get_backend_version() >= 91100) && (detail::get_backend_version() < 91300)) {
+
+ if ((128 < d_qk) && (d_qk <= 192) && (64 < d_v) && (d_v <= 128)) {
+
+ // DeepSeek case, 9.11 only supports 192 hidden dim
+ RETURN_CUDNN_FRONTEND_ERROR_IF( (d_v != 128) && (d_qk != 192),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Num hidden_dim d_v should be equal to 128 if d_qk is 192");
+ }
+ }
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF((d_qk > 256) || (d_qk % 8 != 0) || (d_v > 256) || (d_v % 8 != 0),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Num hidden_dim should be less than or equal to 256 and hidden_dim should be multiple of 8");
+
+ } else if (prop_major == 10 && detail::get_backend_version() >= 91100) {
+ // validate basic dimension requirements
+ if (d_qk == 192) { // special case for 192 hidden dim
+ RETURN_CUDNN_FRONTEND_ERROR_IF( (d_v != 128),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Num hidden_dim d_v should be equal to 128 if d_qk is 192");
+ } else if (detail::get_backend_version() >= 92300 && d_qk == 256 && d_v == 256) {
+ is_d256_on_blackwell = true;
+ attributes.is_deterministic_algorithm = true;
+ } else {
+ RETURN_CUDNN_FRONTEND_ERROR_IF((d_qk > 128) || (d_qk % 8 != 0) || (d_v > 128) || (d_v % 8 != 0),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Num hidden_dim should be less than or equal to 128 and hidden_dim should be multiple of 8 when d_qk != d_v");
+ }
+ } else {
+ // validate basic dimension requirements
+ RETURN_CUDNN_FRONTEND_ERROR_IF((d_qk > 128) || (d_qk % 8 != 0) || (d_v > 128) || (d_v % 8 != 0),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Num hidden_dim should be less than or equal to 128 and hidden_dim should be multiple of 8");
+ }
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF((attributes.attention_score_modifier != nullptr) &&
+ (attributes.alibi_mask || attributes.padding_mask || attributes.has_causal_like_masking() ||
+ attributes.left_bound.has_value()), error_code_t::GRAPH_NOT_SUPPORTED,"Attention score mod enabled and hence other subgraphs are disabled.");
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF((h_q % h_k != 0) || (h_q % h_v != 0),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "For group-query attention, number of heads for key and query must be a factor of number of heads for query");
+
+ // validate options for attn_scale
+ auto const& attn_scale = attributes.inputs.find(input_names::Attn_scale);
+ bool const has_attn_scale = (attn_scale != attributes.inputs.end()) && (attn_scale->second != nullptr);
+ RETURN_CUDNN_FRONTEND_ERROR_IF(has_attn_scale && attributes.attn_scale_value.has_value(),
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "attn_scale with tensor and value cannot be set at the same time.");
+
+ // validate alibi requirements
+ RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.alibi_mask && !(attributes.right_bound.has_value() && attributes.right_bound.value() == 0),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "When alibi mask is used, diagonal_band_right_bound needs to be set to 0.");
+
+ // validate options for bias mask
+ RETURN_CUDNN_FRONTEND_ERROR_IF(is_bias && (bias_mask->second->get_data_type() == DataType_t::BOOLEAN),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Bias mask data type cannot be boolean");
+
+ if (s_kv % 128 != 0 && attributes.padding_mask == false && is_ragged == false && detail::get_backend_version() <= 91500) {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Workaround padding mask is enabled for s_q % 128 != 0 and use_padding_mask == false and is_ragged == false");
+ has_workaround_padding_mask = true;
+ batch_size_for_workaround_padding_mask = attributes.inputs.at(input_names::Q)->get_dim()[0];
+ s_q_for_workaround_padding_mask = s_q;
+ s_kv_for_workaround_padding_mask = s_kv;
+ workaround_padding_mask_seq_len_q = std::make_shared<Tensor_attributes>();
+ workaround_padding_mask_seq_len_q->set_name("workaround_padding_mask_seq_len_q").set_dim({batch_size_for_workaround_padding_mask,1,1,1}).set_stride({1,1,1,1}).set_data_type(DataType_t::INT32);
+ workaround_padding_mask_seq_len_kv = std::make_shared<Tensor_attributes>();
+ workaround_padding_mask_seq_len_kv->set_name("workaround_padding_mask_seq_len_kv").set_dim({batch_size_for_workaround_padding_mask,1,1,1}).set_stride({1,1,1,1}).set_data_type(DataType_t::INT32);
+ attributes.set_padding_mask(true);
+ attributes.set_seq_len_q(workaround_padding_mask_seq_len_q).set_seq_len_kv(workaround_padding_mask_seq_len_kv);
+ }
+
+ // validate options for padding mask
+ auto const& seq_len_q = attributes.inputs.find(input_names::SEQ_LEN_Q);
+ bool const has_seq_len_q = (seq_len_q != attributes.inputs.end()) && (seq_len_q->second != nullptr);
+ auto const& seq_len_kv = attributes.inputs.find(input_names::SEQ_LEN_KV);
+ bool const has_seq_len_kv = (seq_len_kv != attributes.inputs.end()) && (seq_len_kv->second != nullptr);
+ RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.padding_mask && (!has_seq_len_q || !has_seq_len_kv),
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "Padding mask requires seq_len_q and seq_len_kv to be set.");
+ RETURN_CUDNN_FRONTEND_ERROR_IF((!attributes.padding_mask && !attributes.attention_score_modifier) && (has_seq_len_q || has_seq_len_kv),
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "seq_len_q and seq_len_kv needs to be set only if padding mask is enabled.");
+
+ // validate options for max_total_seq_len
+ RETURN_CUDNN_FRONTEND_ERROR_IF((attributes.max_total_seq_len_q.has_value() || attributes.max_total_seq_len_kv.has_value()) && !is_ragged,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "max_total_seq_len_q is only supported with packed layout");
+
+ // validate options for bottom right causal mask
+ RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.has_causal_mask_bottom_right() && (!attributes.padding_mask) && s_q > s_kv,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Bottom right causal mask does not support max_s_q > max_s_kv. Please virtually slice the Q tensor and pass it as max_s_q == max_s_kv");
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.has_causal_mask_bottom_right() && (is_bias || attributes.alibi_mask || (is_ragged && !attributes.padding_mask) || is_dropout),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Bottom right causal mask is only supported with is_bias=False, is_alibi=False, is_dropout=False. Further is_ragged==True is only allowed when padding_mask=True.");
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.has_causal_mask_bottom_right() && (detail::get_backend_version() < 90600) && ((s_q % 64 != 0) || (s_kv % 64 != 0)),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Bottom right causal mask is only supported with s_q multiple of 64, and s_kv multiple of 64, for cudnn version below 9.6.0");
+
+ // validate options for sliding window length
+ RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.left_bound.has_value() && attributes.left_bound.value() <= 0,
+ error_code_t::INVALID_VALUE,
+ "Left bound (Sliding window length) should be greater than or equals to zero when set.");
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.left_bound.has_value() && (s_q * attributes.left_bound.value() == s_kv * attributes.left_bound.value()) && (detail::get_backend_version() <= 90900) && (prop_major == 9) && attributes.has_causal_mask_bottom_right(),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "On Hopper architecture, this specific combination of s_q, s_kv, and left_bound + right_bound + bottom right diagonal alignment is not supported for backend version 9.9 or below");
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.left_bound.has_value() && (!attributes.padding_mask) && s_q > s_kv,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Sliding window attention is only supported with max_s_q <= max_s_kv.");
+
+ if ((detail::get_backend_version() >= 91002)) {
+ RETURN_CUDNN_FRONTEND_ERROR_IF((attributes.left_bound.has_value() || attributes.right_bound.has_value()) && ((is_ragged && !attributes.padding_mask)),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Left and right bounds with is_ragged==True is only allowed when padding_mask=True. And the diagonal alignment must be set.");
+ } else {
+ RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.left_bound.has_value() && (! attributes.has_causal_like_masking() || is_dropout || is_bias || (is_ragged && !attributes.padding_mask)),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Left and right bounds are only supported with is_dropout=False, is_bias=False. Further is_ragged==True is only allowed when padding_mask=True. Lastly the diagonal alignment must be set.");
+ }
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.right_bound.has_value() && attributes.right_bound.value() < 0,
+ error_code_t::INVALID_VALUE,
+ "Right bound needs to be larger than or equal to zero");
+
+ // validate options for dropout mask
+ RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.dropout_probability.has_value() && is_dropout_custom,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "Using both, custom dropout mask and internal-mask generation using dropout probability, is ill-formed.");
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.dropout_probability.has_value() && attributes.dropout_probability.value() == 1.0,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "Dropout probability cannot be 1 as corresponding scale wont be well formed.");
+
+ // validate options for deterministic algorithm
+ if(attributes.is_deterministic_algorithm && (prop_major == 10)) {
+ RETURN_CUDNN_FRONTEND_ERROR_IF( (detail::get_backend_version() < 91800),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Deterministic algorithm is not supported on blackwell architecture with cudnn version below 9.18.0");
+
+ // dbias bias rng/dropout alibi
+ RETURN_CUDNN_FRONTEND_ERROR_IF(is_dbias || is_rng || is_dropout || attributes.alibi_mask,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Deterministic algorithm is not supported on blackwell architecture when dbias, rng/dropout, alibi is enabled");
+
+ is_deterministic_algorithm_supported_on_blackwell = true;
+ }
+
+ if(detail::get_backend_version() >= 91801) {
+ RETURN_CUDNN_FRONTEND_ERROR_IF(is_ragged && (8 == prop_major || 12 == prop_major) && attributes.is_deterministic_algorithm,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Deterministic algorithm is not supported for bprop thd on SM8X and SM12X GPUs");
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(is_ragged && (8 == prop_major || 12 == prop_major) && attributes.inputs[input_names::Stats]->get_ragged_offset(),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Packed/ragged LSE is not supported for bprop thd on SM8X and SM12X GPUs");
+ }
+
+ // version specific validation
+ RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 90500 && is_dbias && attributes.padding_mask,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "For cuDNN version below 9.5.0, dBias with variable sequence lengths is not supported");
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 90500 && is_dbias && ((s_q % 64 != 0) || (s_kv % 64 != 0)),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "For cuDNN version below 9.5.0, dBias not support s_q/s_kv which aren't multiple of 64");
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 90600 && is_ragged && ((h_q != h_k) || (h_q != h_v)),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "For cuDNN version below 9.6.0, group-query attention with raggged offset is not supported");
+
+ // TODO add version check once fixed
+ RETURN_CUDNN_FRONTEND_ERROR_IF(prop_major == 10 && is_ragged && is_dbias,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "dbias with ragged is not supported for SM Major version 10");
+
+ // validate that datatype is set for the graph
+ RETURN_CUDNN_FRONTEND_ERROR_IF(this->context.get_intermediate_data_type() == DataType_t::NOT_SET,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "Intermediate tensor data type needs to be set as internal tensors require it.");
+ // If dsink is set, sink also needs to be set
+ RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.outputs.find(output_names::DSINK_TOKEN) != attributes.outputs.end() && attributes.inputs.find(input_names::SINK_TOKEN) == attributes.inputs.end(),
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "If dsink is set, sink also needs to be set.");
+ // clang-format on
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ infer_properties_node() override final {
+ // If a 1-D length/offset index tensor [n] is supplied, promote it to the 4-D
+ // [n, 1, 1, 1] (contiguous) form the cuDNN backend requires.
+ promote_index_tensors_to_4d();
+
+ // clang-format off
+ if (detail::get_backend_version() < 90600 && (attributes.max_total_seq_len_q.has_value() || attributes.max_total_seq_len_kv.has_value())) {
+ CUDNN_FE_LOG_LABEL_ENDL("WARNING: sdpa_backward.attributes.max_total_seq_len has been set, but cuDNN version is below 9.6.0 does not support max_total_seq_len_q. The workspace memory size required to execute this graph may be unexpectedly large");
+ attributes.max_total_seq_len_q.reset();
+ attributes.max_total_seq_len_kv.reset();
+ }
+
+ // TODO add version check once fixed
+ int64_t d_qk = attributes.inputs.at(input_names::Q)->get_dim()[3];
+ int64_t d_v = attributes.inputs.at(input_names::V)->get_dim()[3];
+ if ((attributes.max_total_seq_len_q.has_value() || attributes.max_total_seq_len_kv.has_value()) && (d_qk % 16 != 0 || d_v % 16 != 0)) {
+ CUDNN_FE_LOG_LABEL_ENDL("WARNING: sdpa_backward.attributes.max_total_seq_len has been set, but d is not a multiple of 16 has a known functional issue. The workspace memory size required to execute this graph may be unexpectedly large");
+ attributes.max_total_seq_len_q.reset();
+ attributes.max_total_seq_len_kv.reset();
+ }
+
+
+ if(detail::get_backend_version() >= 91801) {
+ int32_t const prop_major = context.get_sm_version() / 10;
+ if(8 == prop_major || 12 == prop_major) {
+ if(attributes.max_total_seq_len_q.has_value() || attributes.max_total_seq_len_kv.has_value()) {
+ attributes.max_total_seq_len_q.reset();
+ attributes.max_total_seq_len_kv.reset();
+ CUDNN_FE_LOG_LABEL_ENDL("WARNING: sdpa_backward.attributes.max_total_seq_len has been set, but ampere style kernels have a known functional issue. The workspace memory size required to execute this graph may be unexpectedly large");
+
+ }
+ }
+ }
+ // clang-format on
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ expand_node() override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for CompositeSDPABackwardNode " << attributes.name);
+
+ attributes.fill_from_context(context);
+
+ // Gather dim to fill properties of virtual tensors
+ auto const& q_dim = attributes.inputs[input_names::Q]->get_dim();
+ auto b = q_dim[0];
+ auto h_q = q_dim[1];
+ auto s_q = q_dim[2];
+ auto d_qk = q_dim[3];
+ auto const& k_dim = attributes.inputs[input_names::K]->get_dim();
+ auto h_k = k_dim[1];
+ auto s_kv = k_dim[2];
+ auto const& v_dim = attributes.inputs[input_names::V]->get_dim();
+ auto h_v = v_dim[1];
+ auto d_v = v_dim[3];
+
+ // cuDNN frontend API attention requires Q, K, V where
+ // Q = {b, h_q, s_q, d_qk}
+ // K = {b, h_k, s_kv, d_qk}
+ // V = {b, h_v, s_kv, d_v}
+ // but cuDNN backend API attention requires Q, KT, VT
+ // Q = {b, h_q, s_q, d_qk}
+ // KT = {b, h_k, d_qk, s_kv}
+ // VT = {b, h_v, d_v, s_kv}
+ // So the code below maps the K->KT and V->VT
+ std::vector<int64_t> temp_vec;
+
+ temp_vec = attributes.inputs[input_names::K]->get_dim();
+ std::swap(temp_vec[2], temp_vec[3]);
+ attributes.inputs[input_names::K]->set_dim(temp_vec);
+
+ temp_vec = attributes.inputs[input_names::K]->get_stride();
+ std::swap(temp_vec[2], temp_vec[3]);
+ attributes.inputs[input_names::K]->set_stride(temp_vec);
+
+ temp_vec = attributes.inputs[input_names::V]->get_dim();
+ std::swap(temp_vec[2], temp_vec[3]);
+ attributes.inputs[input_names::V]->set_dim(temp_vec);
+
+ temp_vec = attributes.inputs[input_names::V]->get_stride();
+ std::swap(temp_vec[2], temp_vec[3]);
+ attributes.inputs[input_names::V]->set_stride(temp_vec);
+
+ std::shared_ptr<Tensor_attributes> last_output, exp_s_output, dS_output, rng_output;
+
+ // --------------Initialize and create tensors before creating nodes--------------------
+ // one_tensor is needed for non-dropout graphs
+ // one_tensor is passed by the node
+ auto one_tensor = std::make_shared<Tensor_attributes>(1.0f);
+
+ if (attributes.attn_scale_value.has_value()) {
+ attributes.inputs[input_names::Attn_scale] =
+ std::make_shared<Tensor_attributes>(attributes.attn_scale_value.value());
+ }
+
+ // if dropout_mask is used, then the user passes scale and scale_inverse
+ bool is_dropout_prob = (attributes.dropout_probability.has_value());
+ bool is_dropout_mask = (attributes.inputs[input_names::Dropout_mask] != nullptr);
+ if (is_dropout_prob) {
+ float dropout_scale_value = 1.0f / (1.0f - attributes.dropout_probability.value());
+ float dropout_scale_inv_value = (1.0f - attributes.dropout_probability.value());
+
+ attributes.inputs[input_names::Dropout_scale] = std::make_shared<Tensor_attributes>(dropout_scale_value);
+ attributes.inputs[input_names::Dropout_scale_inv] =
+ std::make_shared<Tensor_attributes>(dropout_scale_inv_value);
+ }
+
+ // ---------------------input tensor workarounds---------------------------
+
+ bool use_dp_workspace = false;
+
+ int32_t const prop_major = context.get_sm_version() / 10;
+
+ if (detail::get_backend_version() >= 8905 && detail::get_backend_version() < 90000) {
+ // workspace optimization is enabled by default when:
+ // 8.9.5 <= cudnn version < 9.0.0
+ // device >= hopper
+ // batch * num_heads * seq_len_q * seq_len_kv * 2 <= dP workspace limit
+ //
+ // This following environment variable allows you to control the dP workspace limit.
+ // From cuDNN version 9.0.0, this option is obsolete will be ignored.
+ // CUDNN_FRONTEND_ATTN_DP_WORKSPACE_LIMIT=unset - enable workspace opt. until the default 256MB limit.
+ // CUDNN_FRONTEND_ATTN_DP_WORKSPACE_LIMIT=-1 - always enable workspace opt.
+ // CUDNN_FRONTEND_ATTN_DP_WORKSPACE_LIMIT=0 - always disable workspace opt.
+ // CUDNN_FRONTEND_ATTN_DP_WORKSPACE_LIMIT=n - enable workspace opt. until the n byte limit
+
+ // hopper or above
+ if (prop_major >= 9) {
+ // default upper limit for workspace 256MB
+ int64_t max_dp_workspace_bytes = 256 * 1024 * 1024;
+
+ // allow setting the upper limit with envvars
+ const char* env_dp_workspace_limit_char = get_environment("CUDNN_FRONTEND_ATTN_DP_WORKSPACE_LIMIT");
+ if (env_dp_workspace_limit_char) {
+ char* end_ptr = nullptr;
+ max_dp_workspace_bytes = std::strtoll(env_dp_workspace_limit_char, &end_ptr, 10);
+
+ if (*end_ptr != '\0') {
+ RETURN_CUDNN_FRONTEND_ERROR_IF(true,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "Invalid argument for CUDNN_FRONTEND_ATTN_DP_WORKSPACE_LIMIT "
+ "(int64_t; in bytes)");
+ }
+ }
+
+ int64_t workspace_s_q = ((s_q + 64 - 1) / 64) * 64;
+ int64_t workspace_s_kv = ((s_kv + 64 - 1) / 64) * 64;
+ int64_t required_dp_workspace_bytes = b * h_q * workspace_s_q * workspace_s_kv * 2;
+
+ if (max_dp_workspace_bytes == -1) {
+ use_dp_workspace = true;
+ } else if (max_dp_workspace_bytes == 0) {
+ use_dp_workspace = false;
+ } else {
+ use_dp_workspace = (required_dp_workspace_bytes <= max_dp_workspace_bytes);
+ }
+ }
+ }
+
+ // Force dP workspace implementation if:
+ // - dBias is enabled (dBias is only supported on workspace implementation)
+ // - the user force requests deterministic algorithm on hopper
+ if (attributes.outputs[output_names::dBias] || attributes.is_deterministic_algorithm) {
+ use_dp_workspace = true;
+ }
+
+ // --------------RNG node--------------------
+
+ if (is_dropout_prob) {
+ if (attributes.outputs[output_names::RNG_DUMP] != nullptr) {
+ rng_output = attributes.outputs[output_names::RNG_DUMP];
+ rng(attributes.inputs[input_names::Seed],
+ attributes.inputs[input_names::Offset],
+ Rng_attributes()
+ .set_name("rng")
+ .set_distribution(RngDistribution_t::BERNOULLI)
+ .set_bernoulli_probability(1.0f - attributes.dropout_probability.value()),
+ rng_output);
+ } else {
+ rng_output = rng(attributes.inputs[input_names::Seed],
+ attributes.inputs[input_names::Offset],
+ Rng_attributes()
+ .set_name("rng")
+ .set_distribution(RngDistribution_t::BERNOULLI)
+ .set_bernoulli_probability(1.0f - attributes.dropout_probability.value()));
+ rng_output->set_dim({b, h_q, s_q, s_kv}).set_stride({h_q * s_q * s_kv, s_q * s_kv, s_kv, 1});
+ }
+ } else if (is_dropout_mask) {
+ rng_output = attributes.inputs[input_names::Dropout_mask];
+ }
+
+ // --------------"dO * o => softmax_sum" chain--------------------
+
+ // last_output = dO * O
+ last_output = pointwise(attributes.inputs[input_names::dO],
+ attributes.inputs[input_names::O],
+ Pointwise_attributes().set_name("mul_dO_O").set_mode(PointwiseMode_t::MUL));
+ last_output->set_dim({b, h_q, s_q, d_v}).set_stride({h_q * s_q * d_v, s_q * d_v, h_q * d_v, 1});
+
+ // last_output = reduce(last_output, "b hq sq dv -> b hq sq 1")
+ last_output =
+ reduction(last_output, Reduction_attributes().set_name("reduce_dO_o").set_mode(ReductionMode_t::ADD));
+ last_output->set_dim({b, h_q, s_q, 1}).set_stride({h_q * s_q, s_q, 1, 1});
+
+ if (attributes.outputs.find(output_names::DSINK_TOKEN) != attributes.outputs.end()) {
+ // sub_sink = sink - stats
+ auto sub_sink = pointwise(attributes.inputs[input_names::SINK_TOKEN],
+ attributes.inputs[input_names::Stats],
+ Pointwise_attributes().set_name("sub_sink").set_mode(PointwiseMode_t::SUB));
+
+ // exp_sink = exp(sub_sink)
+ auto exp_sink =
+ pointwise(sub_sink, Pointwise_attributes().set_name("exp_sink").set_mode(PointwiseMode_t::EXP));
+
+ // per_token_grad = exp_sink * last_output
+ auto per_token_grad =
+ pointwise(exp_sink,
+ last_output,
+ Pointwise_attributes().set_name("mul_exp_sink_last_output").set_mode(PointwiseMode_t::MUL));
+
+ // dSink = redduce(per_token_grad)
+ reduction(per_token_grad,
+ Reduction_attributes().set_name("reduce_per_token_grad").set_mode(ReductionMode_t::ADD),
+ attributes.outputs[output_names::DSINK_TOKEN]);
+ }
+
+ // softmax_sum = last_output * dropout_scale
+ last_output = pointwise(last_output,
+ attributes.inputs[input_names::Dropout_scale_inv]
+ ? attributes.inputs[input_names::Dropout_scale_inv]
+ : one_tensor,
+ Pointwise_attributes().set_name("scale_dropout_inv").set_mode(PointwiseMode_t::MUL));
+ last_output->set_dim({b, h_q, s_q, 1}).set_stride({h_q * s_q, s_q, 1, 1});
+
+ softmax_sum = last_output;
+ softmax_sum->set_is_virtual(false);
+ softmax_sum->set_dim({b, h_q, s_q, 1});
+ softmax_sum->set_data_type(DataType_t::FLOAT);
+
+ if (attributes.inputs[input_names::Stats]->get_ragged_offset() && attributes.max_total_seq_len_q.has_value()) {
+ // sized TH1 softmax_sum
+ softmax_sum->set_stride(attributes.inputs[input_names::Stats]->get_stride());
+ softmax_sum->set_ragged_offset(attributes.inputs[input_names::Stats]->get_ragged_offset());
+ softmax_sum_size = attributes.max_total_seq_len_q.value() *
+ (attributes.inputs[input_names::Stats]->get_stride())[2] * sizeof(float);
+ } else {
+ // sized BHS1 softmax_sum
+ softmax_sum->set_stride({h_q * s_q, s_q, 1, 1});
+ softmax_sum_size = b * h_q * s_q * 1 * sizeof(float);
+ }
+
+ // --------------"Q @ KT => exp_softmax => dV" chain--------------------
+
+ // s = einsum(q, k, "b hq sq dqk, b (hk g) skv dqk -> b hq sq skv", g=hq//hk)
+ last_output = matmul(attributes.inputs[input_names::Q],
+ attributes.inputs[input_names::K],
+ Matmul_attributes()
+ .set_name("matmul_Q_KT")
+ .set_m_override(attributes.inputs[input_names::SEQ_LEN_Q])
+ .set_n_override(attributes.inputs[input_names::SEQ_LEN_KV]));
+ last_output->set_dim({b, h_q, s_q, s_kv}).set_stride({h_q * s_q * s_kv, s_q * s_kv, s_kv, 1});
+
+ // last_output = last_output * attention_scale
+ if (attributes.inputs[input_names::Attn_scale]) {
+ last_output = pointwise(last_output,
+ attributes.inputs[input_names::Attn_scale],
+ Pointwise_attributes().set_name("mul_s_attn_scale").set_mode(PointwiseMode_t::MUL));
+ }
+
+ if (attributes.attention_score_modifier != nullptr) {
+ auto graph_ = std::make_shared<Graph>();
+ std::shared_ptr<INode> node_ = std::static_pointer_cast<INode>(graph_);
+ node_->context = context;
+ last_output = attributes.attention_score_modifier(graph_, last_output);
+ sub_nodes.emplace_back(node_);
+ }
+
+ // (optional) last_output = last_output + bias
+ if (attributes.inputs.find(input_names::Bias) != attributes.inputs.end() &&
+ attributes.inputs[input_names::Bias]) {
+ auto graph_ = std::make_shared<Graph>();
+ std::shared_ptr<INode> node_ = std::static_pointer_cast<INode>(graph_);
+ node_->context = context;
+ last_output = attn::score_modifiers::bias(graph_, last_output, attributes.inputs[input_names::Bias]);
+ sub_nodes.emplace_back(node_);
+ }
+
+ // (optional) last_output = last_output + alibi_mask
+ if (attributes.alibi_mask) {
+ auto graph_ = std::make_shared<Graph>();
+ std::shared_ptr<INode> node_ = std::static_pointer_cast<INode>(graph_);
+ node_->context = context;
+ last_output = attn::score_modifiers::alibi_mask(graph_, last_output, alibi_slopes, h_q, alibi_slopes_size);
+ sub_nodes.emplace_back(node_);
+ }
+
+ // last_output = last_output - stats
+ last_output = pointwise(last_output,
+ attributes.inputs[input_names::Stats],
+ Pointwise_attributes().set_name("sub_s_m").set_mode(PointwiseMode_t::SUB));
+
+ // (optional) Apply padding mask
+ if (attributes.padding_mask) {
+ auto graph_ = std::make_shared<Graph>();
+ std::shared_ptr<INode> node_ = std::static_pointer_cast<INode>(graph_);
+ node_->context = context;
+ last_output = attn::score_modifiers::padding_mask(graph_,
+ last_output,
+ attributes.inputs[input_names::SEQ_LEN_KV],
+ attributes.inputs[input_names::SEQ_LEN_Q]);
+ sub_nodes.emplace_back(node_);
+ }
+
+ // Explicitly put the padding value again after the stats have been loaded
+ if (attributes.padding_mask && detail::get_backend_version() >= 90000 &&
+ detail::get_backend_version() < 91000) {
+ auto row_idx_output = pointwise(last_output,
+ Pointwise_attributes()
+ .set_name("gen_row_idx_2nd_padding")
+ .set_mode(PointwiseMode_t::GEN_INDEX)
+ .set_axis(2)
+ .set_compute_data_type(DataType_t::INT32));
+ row_idx_output->set_data_type(DataType_t::INT32);
+
+ auto col_idx_output = pointwise(last_output,
+ Pointwise_attributes()
+ .set_name("gen_col_idx_2nd_padding")
+ .set_mode(PointwiseMode_t::GEN_INDEX)
+ .set_axis(3)
+ .set_compute_data_type(DataType_t::INT32));
+ col_idx_output->set_data_type(DataType_t::INT32);
+
+ auto row_mask_output = pointwise(row_idx_output,
+ attributes.inputs[input_names::SEQ_LEN_Q],
+ Pointwise_attributes()
+ .set_name("lt_row_sq_2nd_padding")
+ .set_mode(PointwiseMode_t::CMP_LT)
+ .set_compute_data_type(DataType_t::BOOLEAN));
+ row_mask_output->set_data_type(DataType_t::BOOLEAN);
+
+ auto col_mask_output = pointwise(col_idx_output,
+ attributes.inputs[input_names::SEQ_LEN_KV],
+ Pointwise_attributes()
+ .set_name("lt_col_skv_2nd_padding")
+ .set_mode(PointwiseMode_t::CMP_LT)
+ .set_compute_data_type(DataType_t::BOOLEAN));
+ col_mask_output->set_data_type(DataType_t::BOOLEAN);
+
+ auto padding_mask_output = pointwise(row_mask_output,
+ col_mask_output,
+ Pointwise_attributes()
+ .set_name("and_row_col_2nd_padding")
+ .set_mode(PointwiseMode_t::LOGICAL_AND)
+ .set_compute_data_type(DataType_t::BOOLEAN));
+ padding_mask_output->set_data_type(DataType_t::BOOLEAN);
+ auto negative_inf_padding =
+ std::make_shared<Tensor_attributes>(attn::score_modifiers::get_negative_inf_value());
+
+ last_output = pointwise(
+ last_output,
+ negative_inf_padding,
+ padding_mask_output,
+ Pointwise_attributes().set_name("select_2nd_padding").set_mode(PointwiseMode_t::BINARY_SELECT));
+ }
+
+ // Apply (bottom-right) causal masking (with right bound) and/or set the left bound
+ if (attributes.left_bound.has_value() || attributes.right_bound.has_value()) {
+ auto graph_ = std::make_shared<Graph>();
+ std::shared_ptr<INode> node_ = std::static_pointer_cast<INode>(graph_);
+ node_->context = context;
+
+ auto s_kv_ptr = attributes.inputs.find(input_names::SEQ_LEN_KV) != attributes.inputs.end()
+ ? attributes.inputs[input_names::SEQ_LEN_KV]
+ : nullptr;
+ auto s_q_ptr = attributes.inputs.find(input_names::SEQ_LEN_Q) != attributes.inputs.end()
+ ? attributes.inputs[input_names::SEQ_LEN_Q]
+ : nullptr;
+
+ last_output = attn::score_modifiers::sliding_window_mask(graph_,
+ last_output,
+ attributes.diagonal_alignment,
+ attributes.left_bound,
+ attributes.right_bound,
+ s_q,
+ s_kv,
+ s_q_ptr,
+ s_kv_ptr,
+ /*cu_s_q_ptr=*/nullptr,
+ /*cu_s_kv_ptr=*/nullptr);
+ sub_nodes.emplace_back(std::move(node_));
+ }
+
+ // last_output = exp(last_output)
+ last_output = pointwise(last_output, Pointwise_attributes().set_name("exp_s").set_mode(PointwiseMode_t::EXP));
+
+ exp_s_output = last_output;
+
+ // (optional) last_output = last_output * dropout rng_output
+ if (is_dropout_prob || is_dropout_mask) {
+ last_output =
+ pointwise(last_output,
+ rng_output,
+ Pointwise_attributes().set_name("mul_p_dropout_mask").set_mode(PointwiseMode_t::MUL));
+ }
+
+ // (optional) last_output = last_output * dropout_scale
+ if (attributes.inputs[input_names::Dropout_scale]) {
+ last_output =
+ pointwise(last_output,
+ attributes.inputs[input_names::Dropout_scale],
+ Pointwise_attributes().set_name("mul_p_dropout_scale").set_mode(PointwiseMode_t::MUL));
+ }
+
+ // dV = einsum(p, dO, "b hq sq skv", "b hq sq dv -> b hq skv dv")
+ // if GQA, then dV = reduce(dV, "b (hv g) skv dv -> b hv skv dv", g=hq//hv)
+ // as reshape + matmul
+ last_output = reshape(last_output, Reshape_attributes().set_name("reshape_p"));
+ last_output->set_dim({b, h_q, s_kv, s_q}).set_stride({h_q * s_q * s_kv, s_q * s_kv, 1, s_kv});
+ last_output->set_data_type(attributes.inputs[input_names::Q]->get_data_type());
+
+ if (h_q == h_v) {
+ // for MHA
+ matmul(last_output,
+ attributes.inputs[input_names::dO],
+ Matmul_attributes()
+ .set_name("matmul_pT_dO")
+ .set_m_override(attributes.inputs[input_names::SEQ_LEN_KV])
+ .set_k_override(attributes.inputs[input_names::SEQ_LEN_Q]),
+ attributes.outputs[output_names::dV]);
+ } else {
+ // for GQA and MQA
+ dV_fullhead = matmul(last_output,
+ attributes.inputs[input_names::dO],
+ Matmul_attributes()
+ .set_name("matmul_pT_dO")
+ .set_m_override(attributes.inputs[input_names::SEQ_LEN_KV])
+ .set_k_override(attributes.inputs[input_names::SEQ_LEN_Q]));
+
+ dV_fullhead->set_dim({b, h_q, s_kv, d_v});
+ dV_fullhead->set_data_type(attributes.inputs[input_names::Q]->get_data_type());
+
+ if (attributes.outputs[output_names::dV]->get_ragged_offset() &&
+ attributes.max_total_seq_len_kv.has_value()) {
+ // map dV strides to dV_fullhead strides
+ std::vector<int64_t> dV_fullhead_stride = attributes.outputs[output_names::dV]->get_stride();
+ dV_fullhead_stride[2] = dV_fullhead_stride[2] * (h_q / h_v); // sequence stride
+ dV_fullhead_stride[0] = dV_fullhead_stride[0] * (h_q / h_v); // batch stride
+ dV_fullhead->set_stride(dV_fullhead_stride);
+ // map dV ragged offset to dV_fullhead ragged offset with implicit multiplier
+ // implicit multiplier = h_q / h_v
+ dV_fullhead->set_ragged_offset(attributes.outputs[output_names::dV]->get_ragged_offset());
+ // non virtual dV full head
+ dV_fullhead->set_is_virtual(false);
+ dV_fullhead_size = attributes.max_total_seq_len_kv.value() * dV_fullhead_stride[2] * sizeof(float);
+ } else {
+ // sized BHSD dQ_accum
+ dV_fullhead->set_stride({h_q * s_kv * d_v, s_kv * d_v, d_v, 1});
+ }
+
+ reduction(dV_fullhead,
+ Reduction_attributes().set_name("red_dV_head").set_mode(ReductionMode_t::ADD),
+ attributes.outputs[output_names::dV]);
+ }
+
+ // --------------"dO @ VT => dS_output => dK" chain--------------------
+
+ // dP = einsum(dO, v, "b hq sq dv, b (hv g) skv dv -> b hq sq skv", g=hq//hv)
+ last_output = matmul(attributes.inputs[input_names::dO],
+ attributes.inputs[input_names::V],
+ Matmul_attributes()
+ .set_name("matmul_dO_VT")
+ .set_m_override(attributes.inputs[input_names::SEQ_LEN_Q])
+ .set_n_override(attributes.inputs[input_names::SEQ_LEN_KV]));
+ last_output->set_dim({b, h_q, s_q, s_kv}).set_stride({h_q * s_q * s_kv, s_q * s_kv, s_kv, 1});
+
+ // last_output = last_output(dP) * mask
+ if (is_dropout_prob || is_dropout_mask) {
+ last_output = pointwise(last_output,
+ rng_output,
+ Pointwise_attributes().set_name("dP_dropout_mask").set_mode(PointwiseMode_t::MUL));
+ }
+
+ // last_output = last_output - softmax_sum
+ last_output = pointwise(last_output,
+ softmax_sum,
+ Pointwise_attributes().set_name("sub_dP_softmax_sum").set_mode(PointwiseMode_t::SUB));
+
+ // last_output = last_output * exp_s_output
+ last_output = pointwise(
+ last_output, exp_s_output, Pointwise_attributes().set_name("mul_dP_exp_s").set_mode(PointwiseMode_t::MUL));
+
+ // (optional) last_output = last_output * dropout_scale
+ if (attributes.inputs[input_names::Dropout_scale]) {
+ last_output =
+ pointwise(last_output,
+ attributes.inputs[input_names::Dropout_scale],
+ Pointwise_attributes().set_name("mul_dS_dropout_scale").set_mode(PointwiseMode_t::MUL));
+ }
+
+ if (attributes.outputs[output_names::dBias]) {
+ reduction(last_output,
+ Reduction_attributes().set_name("red_dP_dBias").set_mode(ReductionMode_t::ADD),
+ attributes.outputs[output_names::dBias]);
+ }
+
+ // apply the bprop of attention score modifier
+ if (attributes.attention_score_modifier_bprop != nullptr) {
+ auto graph_ = std::make_shared<Graph>();
+ std::shared_ptr<INode> node_ = std::static_pointer_cast<INode>(graph_);
+ node_->context = context;
+ last_output = attributes.attention_score_modifier_bprop(graph_, last_output);
+ sub_nodes.emplace_back(node_);
+ }
+
+ // (optional) last_output = last_output * bmm_scale
+ if (attributes.inputs[input_names::Attn_scale]) {
+ last_output =
+ pointwise(last_output,
+ attributes.inputs[input_names::Attn_scale],
+ Pointwise_attributes().set_name("mul_dS_attn_scale").set_mode(PointwiseMode_t::MUL));
+ }
+
+ dS_output = last_output;
+
+ // dK = einsum(dS, Q, "b hq sq skv", "b hq sq dqk -> b hq skv dqk")
+ // if GQA, then dK = reduce(dK, "b (hk g) skv dqk -> b hk skv dqk", hq//hk)
+ // as reshape + matmul
+ last_output = reshape(last_output, Reshape_attributes().set_name("reshape_dS"));
+ last_output->set_dim({b, h_q, s_kv, s_q}).set_stride({h_q * s_q * s_kv, s_q * s_kv, 1, s_kv});
+ last_output->set_data_type(attributes.inputs[input_names::Q]->get_data_type());
+
+ if (h_q == h_k) {
+ // for MHA
+ matmul(last_output,
+ attributes.inputs[input_names::Q],
+ Matmul_attributes()
+ .set_name("matmul_dST_Q")
+ .set_m_override(attributes.inputs[input_names::SEQ_LEN_KV])
+ .set_k_override(attributes.inputs[input_names::SEQ_LEN_Q]),
+ attributes.outputs[output_names::dK]);
+ } else {
+ // for GQA and MQA
+ dK_fullhead = matmul(last_output,
+ attributes.inputs[input_names::Q],
+ Matmul_attributes()
+ .set_name("matmul_dST_Q")
+ .set_m_override(attributes.inputs[input_names::SEQ_LEN_KV])
+ .set_k_override(attributes.inputs[input_names::SEQ_LEN_Q]));
+
+ dK_fullhead->set_dim({b, h_q, s_kv, d_qk});
+ dK_fullhead->set_data_type(attributes.inputs[input_names::Q]->get_data_type());
+
+ if (attributes.outputs[output_names::dK]->get_ragged_offset() &&
+ attributes.max_total_seq_len_kv.has_value()) {
+ // sized THD dK_full_heads
+ // map dK strides to dK_fullhead strides
+ std::vector<int64_t> dK_fullhead_stride = attributes.outputs[output_names::dK]->get_stride();
+ dK_fullhead_stride[0] = dK_fullhead_stride[0] * (h_q / h_k); // batch stride
+ dK_fullhead_stride[2] = dK_fullhead_stride[2] * (h_q / h_k); // sequence stride
+ dK_fullhead->set_stride(dK_fullhead_stride);
+ // map dK ragged offset to dK_fullhead ragged offset with implicit multiplier
+ // implicit multiplier = h_q / h_k
+ dK_fullhead->set_ragged_offset(attributes.outputs[output_names::dK]->get_ragged_offset());
+ // non virtual dK full head
+ dK_fullhead->set_is_virtual(false);
+ dK_fullhead_size = attributes.max_total_seq_len_kv.value() * dK_fullhead_stride[2] * sizeof(float);
+ } else {
+ // sized BHSD dQ_accum
+ dK_fullhead->set_stride({h_q * s_kv * d_qk, s_kv * d_qk, d_qk, 1});
+ }
+
+ reduction(dK_fullhead,
+ Reduction_attributes().set_name("red_dK_head").set_mode(ReductionMode_t::ADD),
+ attributes.outputs[output_names::dK]);
+ }
+
+ // --------------"dp_scaled @ K => dQ" chain--------------------
+
+ auto const& kt_dim = attributes.inputs[input_names::K]->get_dim();
+ auto const& kt_stride = attributes.inputs[input_names::K]->get_stride();
+
+ // dQ = einsum(dS, K, "b hq sq skv, b (hk g) skv dqk -> b hq sq dqk", g=hq//hk)
+ // as reshape + matmul
+ last_output = reshape(attributes.inputs[input_names::K], Reshape_attributes().set_name("reshape_k"));
+ last_output->set_dim({kt_dim[0], kt_dim[1], kt_dim[3], kt_dim[2]})
+ .set_stride({kt_stride[0], kt_stride[1], kt_stride[3], kt_stride[2]});
+
+ if (attributes.inputs[input_names::K]->get_ragged_offset() != nullptr) {
+ last_output->set_ragged_offset(attributes.inputs[input_names::K]->get_ragged_offset());
+ }
+
+ if (!use_dp_workspace) {
+ dQ_accum = std::make_shared<Tensor_attributes>();
+ dQ_accum->set_is_virtual(false);
+ dQ_accum->set_dim({b, h_q, s_q, d_qk});
+ dQ_accum->set_data_type(DataType_t::FLOAT);
+
+ if (attributes.outputs[output_names::dQ]->get_ragged_offset() &&
+ attributes.max_total_seq_len_q.has_value()) {
+ // sized THD dQ_accum
+ dQ_accum->set_stride(attributes.outputs[output_names::dQ]->get_stride());
+ dQ_accum->set_ragged_offset(attributes.outputs[output_names::dQ]->get_ragged_offset());
+ dQ_accum_size = attributes.max_total_seq_len_q.value() *
+ (attributes.outputs[output_names::dQ]->get_stride())[2] * sizeof(float);
+ } else {
+ // sized BHSD dQ_accum
+ dQ_accum->set_stride({h_q * s_q * d_qk, s_q * d_qk, d_qk, 1});
+ dQ_accum_size = b * h_q * s_q * d_qk * sizeof(float);
+ }
+
+ matmul(dS_output,
+ last_output,
+ Matmul_attributes()
+ .set_name("matmul_dS_K")
+ .set_m_override(attributes.inputs[input_names::SEQ_LEN_Q])
+ .set_k_override(attributes.inputs[input_names::SEQ_LEN_KV]),
+ dQ_accum);
+
+ pointwise(dQ_accum,
+ Pointwise_attributes().set_name("identity_dQ").set_mode(PointwiseMode_t::IDENTITY),
+ attributes.outputs[output_names::dQ]);
+ } else {
+ matmul(dS_output,
+ last_output,
+ Matmul_attributes()
+ .set_name("matmul_dS_K")
+ .set_m_override(attributes.inputs[input_names::SEQ_LEN_Q])
+ .set_k_override(attributes.inputs[input_names::SEQ_LEN_KV]),
+ attributes.outputs[output_names::dQ]);
+ }
+
+ return {error_code_t::OK, ""};
+ }
+
+ std::pair<int64_t, std::unordered_map<KnobType_t, int64_t>>
+ override_heuristics_query() const {
+ int32_t const sm_version = context.get_sm_version();
+ bool const use_new_knobs = detail::get_backend_version() >= 92300;
+ // {128,128} bprop: tileM=3, tileN=2, kernelCfg=2(bprop warp), streamK=0, cgaM=0
+ if (sm_version > 103 && (is_deterministic_algorithm_supported_on_blackwell)) {
+ if (use_new_knobs) {
+ return {17,
+ {{KnobType_t::TILE_M, 3},
+ {KnobType_t::TILE_N, 2},
+ {KnobType_t::KERNEL_CFG, 2},
+ {KnobType_t::STREAM_K, 0},
+ {KnobType_t::TILE_CGA_M, 0},
+ {KnobType_t::STAGES, is_d256_on_blackwell ? 3 : 2}}};
+ } else {
+ return {17, {{KnobType_t::KERNEL_CFG, 31}, {KnobType_t::STAGES, is_d256_on_blackwell ? 3 : 2}}};
+ }
+ } else if (is_deterministic_algorithm_supported_on_blackwell) {
+ if (use_new_knobs) {
+ return {5,
+ {{KnobType_t::TILE_M, 3},
+ {KnobType_t::TILE_N, 2},
+ {KnobType_t::KERNEL_CFG, 2},
+ {KnobType_t::STREAM_K, 0},
+ {KnobType_t::TILE_CGA_M, 0},
+ {KnobType_t::STAGES, is_d256_on_blackwell ? 3 : 2}}};
+ } else {
+ return {5, {{KnobType_t::KERNEL_CFG, 31}, {KnobType_t::STAGES, is_d256_on_blackwell ? 3 : 2}}};
+ }
+ } else {
+ return {-1, {}};
+ }
+ }
+
+ virtual int64_t
+ get_fe_workspace_size_node() const override final {
+ int64_t size = 0;
+
+ size += ((alibi_slopes_size + 15) / 16 * 16); // align alibi slopes memory to 16 bytes
+ size += dQ_accum_size;
+ size += dK_fullhead_size;
+ size += dV_fullhead_size;
+ size += softmax_sum_size;
+
+ if (has_workaround_padding_mask) {
+ size += batch_size_for_workaround_padding_mask * sizeof(int32_t) * 2;
+ }
+
+ return size;
+ }
+
+ virtual error_t
+ collect_tensors_in_workspace_node(
+ std::unordered_map<Tensor_attributes::uid_t, std::tuple<int64_t, int64_t, std::vector<float>>>&
+ workspace_modifications,
+ int64_t& offset) const override final {
+ if (attributes.alibi_mask) {
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(Q, input_names::Q);
+ int64_t const h_q = Q->second->get_dim()[1];
+ auto alibi_slopes_vec = detail::get_alibi_slope(h_q);
+ workspace_modifications.emplace(alibi_slopes->get_uid(), std::make_tuple(0, offset, alibi_slopes_vec));
+ int64_t alibi_slopes_size_padded = ((alibi_slopes_size + 15) / 16 * 16);
+ offset = offset + alibi_slopes_size_padded;
+ }
+
+ if (dQ_accum && !dQ_accum->get_is_virtual()) {
+ if (detail::get_backend_version() < 90600) {
+ // prior to cuDNN 9.6.0, dQ_accum needed to be memset by frontend
+ workspace_modifications.emplace(dQ_accum->get_uid(),
+ std::make_tuple(1, offset, std::vector<float>{(float)dQ_accum_size}));
+ } else {
+ workspace_modifications.emplace(dQ_accum->get_uid(), std::make_tuple(2, offset, std::vector<float>()));
+ }
+ offset = offset + dQ_accum_size;
+ }
+
+ if (dK_fullhead && !dK_fullhead->get_is_virtual()) {
+ workspace_modifications.emplace(dK_fullhead->get_uid(), std::make_tuple(2, offset, std::vector<float>()));
+ offset = offset + dK_fullhead_size;
+ }
+
+ if (dV_fullhead && !dV_fullhead->get_is_virtual()) {
+ workspace_modifications.emplace(dV_fullhead->get_uid(), std::make_tuple(2, offset, std::vector<float>()));
+ offset = offset + dV_fullhead_size;
+ }
+
+ if (softmax_sum && !softmax_sum->get_is_virtual()) {
+ workspace_modifications.emplace(softmax_sum->get_uid(), std::make_tuple(2, offset, std::vector<float>()));
+ offset = offset + softmax_sum_size;
+ }
+
+ if (has_workaround_padding_mask) {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Collecting workaround padding mask tensors with batch size "
+ << batch_size_for_workaround_padding_mask << " with UIDs "
+ << workaround_padding_mask_seq_len_q->get_uid() << " and "
+ << workaround_padding_mask_seq_len_kv->get_uid());
+ std::vector<int32_t> workaround_padding_mask_seq_len_q_vec(batch_size_for_workaround_padding_mask,
+ s_q_for_workaround_padding_mask);
+ std::vector<int32_t> workaround_padding_mask_seq_len_kv_vec(batch_size_for_workaround_padding_mask,
+ s_kv_for_workaround_padding_mask);
+
+ // reinterpret_cast the int32_t vector data to float vector for workspace_modifications
+ std::vector<float> workaround_padding_mask_seq_len_q_vec_float(
+ reinterpret_cast<float*>(workaround_padding_mask_seq_len_q_vec.data()),
+ reinterpret_cast<float*>(workaround_padding_mask_seq_len_q_vec.data()) +
+ batch_size_for_workaround_padding_mask);
+ std::vector<float> workaround_padding_mask_seq_len_kv_vec_float(
+ reinterpret_cast<float*>(workaround_padding_mask_seq_len_kv_vec.data()),
+ reinterpret_cast<float*>(workaround_padding_mask_seq_len_kv_vec.data()) +
+ batch_size_for_workaround_padding_mask);
+
+ workspace_modifications.emplace(workaround_padding_mask_seq_len_q->get_uid(),
+ std::make_tuple(0, offset, workaround_padding_mask_seq_len_q_vec_float));
+ offset = offset + batch_size_for_workaround_padding_mask * sizeof(float);
+ workspace_modifications.emplace(workaround_padding_mask_seq_len_kv->get_uid(),
+ std::make_tuple(0, offset, workaround_padding_mask_seq_len_kv_vec_float));
+ offset = offset + batch_size_for_workaround_padding_mask * sizeof(float);
+ }
+
+ return {error_code_t::OK, ""};
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ j.update(R"({"tag": "SDPA_BWD"})"_json);
+ }
+#endif
+};
+
+class UnifiedSDPANode : public SDPANodeBase<UnifiedSDPANode> {
+ public:
+ UnifiedSDPANode(SDPA_attributes&& attributes_, detail::Context const& context)
+ : SDPANodeBase(std::move(attributes_), context) {}
+
+ Type
+ getType() override final {
+ return Type::SDPA;
+ }
+
+ error_t
+ expand_node() override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for UnifiedSDPANode node " << attributes.name);
+
+ // DO NOT REMOVE
+ // input data type is needed for:
+ // - aType of bmm2
+ // - dropout scale in pre 8.9.3
+ attributes.fill_from_context(this->context);
+
+ //// Optional Attn scale
+ // In case user provided a scalar value, do a fused scalar.
+ if (attributes.attn_scale_value.has_value()) {
+ attributes.inputs[input_names::Attn_scale] =
+ std::make_shared<Tensor_attributes>(attributes.attn_scale_value.value());
+ }
+
+ // Optional attention score modifier
+ if (attributes.attention_score_modifier != nullptr) {
+ if (!subgraph) init_subgraph();
+ subgraph_output = attributes.attention_score_modifier(subgraph, subgraph_output);
+ }
+
+ // Optional bias
+ if (attributes.inputs.find(input_names::Bias) != attributes.inputs.end() &&
+ attributes.inputs[input_names::Bias]) {
+ if (!subgraph) init_subgraph();
+ subgraph_output =
+ attn::score_modifiers::bias(subgraph, subgraph_output, attributes.inputs[input_names::Bias]);
+ }
+
+ // Optional alibi mask
+ if (attributes.alibi_mask) {
+ if (!subgraph) init_subgraph();
+ auto h_q = attributes.inputs[input_names::Q]->get_dim()[1];
+ subgraph_output =
+ attn::score_modifiers::alibi_mask(subgraph, subgraph_output, alibi_slopes, h_q, alibi_slopes_size);
+ }
+
+ // Optional casual masking
+ if (attributes.left_bound.has_value() || attributes.right_bound.has_value()) {
+ if (!subgraph) init_subgraph();
+
+ auto s_q = attributes.inputs[input_names::Q]->get_dim()[2];
+ auto s_kv = infer_s_kv();
+ auto s_kv_ptr = attributes.inputs.find(input_names::SEQ_LEN_KV) != attributes.inputs.end()
+ ? attributes.inputs[input_names::SEQ_LEN_KV]
+ : nullptr;
+ auto s_q_ptr = attributes.inputs.find(input_names::SEQ_LEN_Q) != attributes.inputs.end()
+ ? attributes.inputs[input_names::SEQ_LEN_Q]
+ : nullptr;
+ auto cu_s_kv_ptr = attributes.inputs.find(input_names::CU_SEQ_LEN_KV) != attributes.inputs.end()
+ ? attributes.inputs[input_names::CU_SEQ_LEN_KV]
+ : nullptr;
+ auto cu_s_q_ptr = attributes.inputs.find(input_names::CU_SEQ_LEN_Q) != attributes.inputs.end()
+ ? attributes.inputs[input_names::CU_SEQ_LEN_Q]
+ : nullptr;
+
+ subgraph_output = attn::score_modifiers::sliding_window_mask(subgraph,
+ subgraph_output,
+ attributes.diagonal_alignment,
+ attributes.left_bound,
+ attributes.right_bound,
+ s_q,
+ s_kv,
+ s_q_ptr,
+ s_kv_ptr,
+ cu_s_q_ptr,
+ cu_s_kv_ptr);
+ }
+
+ if (subgraph) {
+ subgraph_output->set_name(attributes.name + "::subgraph_output");
+ auto subgraph_node = std::static_pointer_cast<INode>(subgraph);
+ sub_nodes.emplace_back(subgraph_node);
+ }
+
+ // For BE >= 9.21, express softmax as a UnifiedSoftmaxNode whose backend descriptor
+ // will be set via CUDNN_ATTR_OPERATION_SDPA_FWD_SOFTMAX_DESC.
+ // For older versions, stats (if present) is set directly via CUDNN_ATTR_OPERATION_SDPA_FWD_STATSDESC.
+ auto effective_cudnn_ver = std::min(detail::get_compiled_version(), detail::get_backend_version());
+
+ auto has_output = [&](auto name) {
+ auto it = attributes.outputs.find(name);
+ return it != attributes.outputs.end() && it->second != nullptr;
+ };
+ bool has_softmax_features = has_output(output_names::Stats) || has_output(output_names::Max) ||
+ has_output(output_names::Sum_exp) || attributes.has_sink_token();
+
+ if (effective_cudnn_ver >= 92100 && has_softmax_features) {
+ auto b = attributes.inputs[input_names::Q]->get_dim()[0];
+ auto h_q = attributes.inputs[input_names::Q]->get_dim()[1];
+ auto s_q = attributes.inputs[input_names::Q]->get_dim()[2];
+ auto s_kv = infer_s_kv();
+
+ auto softmax_p = std::make_shared<Tensor_attributes>();
+ softmax_p->set_is_virtual(true);
+ softmax_p->set_name(attributes.name + "::softmax_P");
+ softmax_p->set_dim({b, h_q, s_q, s_kv});
+ softmax_p->set_stride({h_q * s_q * s_kv, s_q * s_kv, s_kv, 1});
+
+ auto softmax_s = std::make_shared<Tensor_attributes>();
+ softmax_s->set_is_virtual(true);
+ softmax_s->set_name(attributes.name + "::softmax_S");
+ softmax_s->set_dim({b, h_q, s_q, s_kv});
+ softmax_s->set_stride({h_q * s_q * s_kv, s_q * s_kv, s_kv, 1});
+
+ auto softmax_attrs = Softmax_attributes().set_name(attributes.name + "::softmax");
+ softmax_attrs.inputs[Softmax_attributes::input_names::P] = softmax_p;
+ softmax_attrs.outputs[Softmax_attributes::output_names::S] = softmax_s;
+
+ if (attributes.has_sink_token()) {
+ softmax_attrs.set_sink(attributes.inputs[input_names::SINK_TOKEN]);
+ }
+ if (has_output(output_names::Stats)) {
+ softmax_attrs.outputs[Softmax_attributes::output_names::Stats] =
+ attributes.outputs[output_names::Stats];
+ }
+ if (has_output(output_names::Max)) {
+ softmax_attrs.outputs[Softmax_attributes::output_names::Max] = attributes.outputs[output_names::Max];
+ }
+ if (has_output(output_names::Sum_exp)) {
+ softmax_attrs.outputs[Softmax_attributes::output_names::Sum_exp] =
+ attributes.outputs[output_names::Sum_exp];
+ }
+
+ softmax_node = std::make_shared<UnifiedSoftmaxNode>(std::move(softmax_attrs), context);
+ sub_nodes.emplace_back(softmax_node);
+ }
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ create_cudnn_operations(
+ std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations,
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations,
+ managed_backend_descriptor_t& raw_operations,
+ std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final {
+ CUDNN_FRONTEND_UNUSED(operations);
+ CUDNN_FE_LOG_LABEL("INFO: Building UnifiedSDPANode operations " << attributes.name << " ");
+ auto cudnn_ver_error = error_t{error_code_t::GRAPH_NOT_SUPPORTED, "Unified SDPA node requires cuDNN 9.13.1"};
+
+#if (CUDNN_VERSION >= 91301)
+ NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(91301, cudnn_ver_error);
+ auto unified_sdpa_operation =
+ make_shared_backend_pointer((cudnnBackendDescriptorType_t)CUDNN_BACKEND_OPERATION_SDPA_FWD_DESCRIPTOR);
+
+ auto Q = attributes.inputs.find(SDPA_attributes::input_names::Q)->second;
+ auto backend_q = tensors[Q->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_SDPA_FWD_QDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_q));
+
+ auto K = attributes.inputs.find(SDPA_attributes::input_names::K)->second;
+ auto backend_k = tensors[K->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_SDPA_FWD_KDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_k));
+
+ auto V = attributes.inputs.find(SDPA_attributes::input_names::V)->second;
+ auto backend_v = tensors[V->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_SDPA_FWD_VDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_v));
+
+ auto O = attributes.outputs.find(SDPA_attributes::output_names::O)->second;
+ auto backend_o = tensors[O->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_SDPA_FWD_ODESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_o));
+
+ if (softmax_node) {
+ auto softmax_ver_error =
+ error_t{error_code_t::GRAPH_NOT_SUPPORTED, "SOFTMAX_DESC in unified SDPA node requires cuDNN 9.21.0"};
+#if (CUDNN_VERSION >= 92100)
+ NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(92100, softmax_ver_error);
+
+ managed_backend_descriptor_t softmax_raw_ops;
+ std::unordered_set<Tensor_attributes::uid_t> softmax_uids;
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>> softmax_ops;
+ CHECK_CUDNN_FRONTEND_ERROR(
+ softmax_node->create_cudnn_operations(softmax_uids, softmax_ops, softmax_raw_ops, tensors));
+
+ auto backend_softmax = softmax_raw_ops[0]->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_SDPA_FWD_SOFTMAX_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_softmax));
+
+ uids_involved_in_operations.insert(softmax_uids.begin(), softmax_uids.end());
+#else
+ return softmax_ver_error;
+#endif
+ } else {
+ auto stats_it = attributes.outputs.find(SDPA_attributes::output_names::Stats);
+ if (stats_it != attributes.outputs.end() && stats_it->second) {
+ auto backend_stats = tensors[stats_it->second->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_SDPA_FWD_STATSDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_stats));
+ }
+ }
+
+ auto attn_scale_it = attributes.inputs.find(SDPA_attributes::input_names::Attn_scale);
+ if (attn_scale_it != attributes.inputs.end()) {
+ auto backend_scale = tensors[attn_scale_it->second->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_SDPA_FWD_SCALEDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_scale));
+ }
+
+ auto block_mask_it = attributes.inputs.find(SDPA_attributes::input_names::Block_mask);
+ if (block_mask_it != attributes.inputs.end() && block_mask_it->second != nullptr) {
+ auto block_mask_cudnn_ver_error =
+ error_t{error_code_t::GRAPH_NOT_SUPPORTED, "Block mask in unified SDPA node requires cuDNN 9.14.0"};
+#if CUDNN_VERSION >= 91400
+ NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(91400, block_mask_cudnn_ver_error);
+ auto backend_block_mask = tensors[block_mask_it->second->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_SDPA_FWD_BLOCK_MASK_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_block_mask));
+#else
+ return block_mask_cudnn_ver_error;
+#endif
+ }
+
+ // Variable-length and paged-attention attributes:
+ // - paged attention K/V page tables (needs cuDNN >= 9.15.0)
+ // - per-batch sequence-length tensors (SEQ_LEN_Q / SEQ_LEN_KV; needs cuDNN >= 9.15.0)
+ // - cumulative sequence-length tensors (CU_SEQ_LEN_Q / CU_SEQ_LEN_KV; needs cuDNN >= 9.24.0).
+ if (is_paged_k() || is_paged_v() || has_seq_len_q() || has_seq_len_kv() || has_cu_seq_len_q() ||
+ has_cu_seq_len_kv()) {
+ auto paged_cudnn_ver_error =
+ error_t{error_code_t::GRAPH_NOT_SUPPORTED,
+ "Paged attention or per-batch sequence lengths in unified SDPA node require cuDNN 9.15.0"};
+#if (CUDNN_VERSION >= 91500)
+ NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(91500, paged_cudnn_ver_error);
+
+ if (is_paged_k()) {
+ auto page_table_K = attributes.inputs.find(SDPA_attributes::input_names::Page_table_K)->second;
+ auto backend_page_table_K = tensors[page_table_K->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_SDPA_FWD_PAGE_TABLE_KDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_page_table_K));
+ }
+
+ if (is_paged_v()) {
+ auto page_table_V = attributes.inputs.find(SDPA_attributes::input_names::Page_table_V)->second;
+ auto backend_page_table_V = tensors[page_table_V->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_SDPA_FWD_PAGE_TABLE_VDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_page_table_V));
+ }
+
+ if (has_seq_len_q()) {
+ auto seq_len_Q = attributes.inputs.find(SDPA_attributes::input_names::SEQ_LEN_Q)->second;
+ auto backend_seq_len_Q = tensors[seq_len_Q->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_SDPA_FWD_SEQ_LEN_QDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_seq_len_Q));
+ }
+
+ if (has_seq_len_kv()) {
+ auto seq_len_KV = attributes.inputs.find(SDPA_attributes::input_names::SEQ_LEN_KV)->second;
+ auto backend_seq_len_KV = tensors[seq_len_KV->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_SDPA_FWD_SEQ_LEN_KVDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_seq_len_KV));
+ }
+
+ if (has_cu_seq_len_q() || has_cu_seq_len_kv()) {
+ auto cu_seq_len_cudnn_ver_error =
+ error_t{error_code_t::GRAPH_NOT_SUPPORTED,
+ "Cumulative sequence length tensors in unified SDPA node require cuDNN 9.24.0"};
+#if (CUDNN_VERSION >= 92400)
+ NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(92400, cu_seq_len_cudnn_ver_error);
+
+ if (has_cu_seq_len_q()) {
+ auto cu_seq_len_Q = attributes.inputs.find(SDPA_attributes::input_names::CU_SEQ_LEN_Q)->second;
+ auto backend_cu_seq_len_Q = tensors[cu_seq_len_Q->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_SDPA_FWD_CU_SEQ_LEN_QDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_cu_seq_len_Q));
+ }
+
+ if (has_cu_seq_len_kv()) {
+ auto cu_seq_len_KV = attributes.inputs.find(SDPA_attributes::input_names::CU_SEQ_LEN_KV)->second;
+ auto backend_cu_seq_len_KV =
+ tensors[cu_seq_len_KV->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_SDPA_FWD_CU_SEQ_LEN_KVDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_cu_seq_len_KV));
+ }
+#else
+ return cu_seq_len_cudnn_ver_error;
+#endif
+ }
+
+ // Ignore attributes.max_seq_len_kv, because unified engine doesn't need it (it's harmless if set).
+
+ // Ignore attributes.padding_mask, because unified engine already applies an implicit padding mask
+ // if either (seq_len_Q and seq_len_KV) or (cu_seq_len_Q and cu_seq_len_KV) are provided.
+ // We already checked in `SDPA_attributes::validate_sdpa_support_surface()` that padding_mask
+ // must be true if and only if one of those pairs is set, so we don't need to check it here.
+#else
+ return paged_cudnn_ver_error;
+#endif
+ }
+
+ // Subgraph attributes
+ if (subgraph) {
+ auto subgraph_cudnn_ver_error = error_t{error_code_t::GRAPH_NOT_SUPPORTED,
+ "Subgraph attributes in unified SDPA node requires cuDNN 9.21.0"};
+#if (CUDNN_VERSION >= 92100)
+ NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(92100, subgraph_cudnn_ver_error);
+
+ CHECK_CUDNN_FRONTEND_ERROR(attn::score_modifiers::build_operation_subgraph(subgraph));
+ auto subgraph_cudnn = std::static_pointer_cast<ICudnn>(subgraph);
+ auto backend_subgraph = subgraph_cudnn->get_operation_graph()->get_raw_desc();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_SDPA_FWD_SUBGRAPH,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_subgraph));
+
+ auto subgraph_input_uid = subgraph_input->get_uid();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_SDPA_FWD_SUBGRAPH_INPUT_UID,
+ CUDNN_TYPE_INT64,
+ 1,
+ &subgraph_input_uid));
+
+ auto subgraph_output_uid = subgraph_output->get_uid();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_SDPA_FWD_SUBGRAPH_OUTPUT_UID,
+ CUDNN_TYPE_INT64,
+ 1,
+ &subgraph_output_uid));
+
+ // Add non-virtual uids from subgraph to uids_involved_in_operations
+ uids_involved_in_operations.insert(subgraph_cudnn->variant_pack_uids.begin(),
+ subgraph_cudnn->variant_pack_uids.end());
+#else
+ return subgraph_cudnn_ver_error;
+#endif
+ }
+
+ // Set unfuse_fma attribute (for SM100: use __fmul_rn + __fadd_rn instead of ffma2 in softmax)
+ if (attributes.unfuse_fma) {
+ auto unfuse_fma_cudnn_ver_error =
+ error_t{error_code_t::GRAPH_NOT_SUPPORTED, "Unfuse FMA in unified SDPA node requires cuDNN 9.21.0"};
+#if CUDNN_VERSION >= 92100
+ NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(92100, unfuse_fma_cudnn_ver_error);
+ bool unfuse_fma_value = true;
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_SDPA_FWD_UNFUSE_FMA,
+ CUDNN_TYPE_BOOLEAN,
+ 1,
+ &unfuse_fma_value));
+#else
+ return unfuse_fma_cudnn_ver_error;
+#endif
+ }
+
+ // Dropout attributes
+ if (attributes.dropout_probability.has_value() && attributes.dropout_probability.value() != 0.0f) {
+ auto dropout_cudnn_ver_error =
+ error_t{error_code_t::GRAPH_NOT_SUPPORTED, "Dropout in unified SDPA node requires cuDNN 9.21.0"};
+#if (CUDNN_VERSION >= 92100)
+ NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(92100, dropout_cudnn_ver_error);
+
+ float dropout_prob = attributes.dropout_probability.value();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_SDPA_FWD_DROPOUT_PROBABILITY,
+ CUDNN_TYPE_FLOAT,
+ 1,
+ &dropout_prob));
+
+ auto seed_it = attributes.inputs.find(SDPA_attributes::input_names::Seed);
+ if (seed_it != attributes.inputs.end() && seed_it->second != nullptr) {
+ auto backend_seed = tensors[seed_it->second->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_SDPA_FWD_DROPOUT_SEED_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_seed));
+ }
+
+ auto offset_it = attributes.inputs.find(SDPA_attributes::input_names::Offset);
+ if (offset_it != attributes.inputs.end() && offset_it->second != nullptr) {
+ auto backend_offset = tensors[offset_it->second->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_SDPA_FWD_DROPOUT_OFFSET_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_offset));
+ }
+
+ auto rng_dump_it = attributes.outputs.find(SDPA_attributes::output_names::RNG_DUMP);
+ if (rng_dump_it != attributes.outputs.end() && rng_dump_it->second != nullptr) {
+ auto backend_rng_dump = tensors[rng_dump_it->second->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_SDPA_FWD_DROPOUT_RNG_DUMP_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_rng_dump));
+ }
+#else
+ return dropout_cudnn_ver_error;
+#endif
+ }
+
+ // FP8 per-tensor scaling attributes (descale Q/K/V/S, scale S/O inputs; amax S/O outputs).
+ if (attributes.mma_core_mode == DataType_t::FP8_E4M3 || attributes.mma_core_mode == DataType_t::FP8_E5M2) {
+ auto fp8_cudnn_ver_error =
+ error_t{error_code_t::GRAPH_NOT_SUPPORTED, "FP8/MXFP8 for the unified SDPA node requires cuDNN 9.30.0"};
+#if (CUDNN_VERSION >= 93000)
+ NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(93000, fp8_cudnn_ver_error);
+
+ // Wire an optional FP8 scaling/descaling tensor (from either the input or output map) to
+ // its backend descriptor, no-op if the caller did not provide it.
+ auto set_fp8_desc = [&](auto const& tensor_map, auto name, cudnnBackendAttributeName_t attr) -> error_t {
+ auto it = tensor_map.find(name);
+ if (it != tensor_map.end() && it->second != nullptr) {
+ auto backend_desc = tensors[it->second->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(),
+ attr,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_desc));
+ }
+ return {error_code_t::OK, ""};
+ };
+
+ CHECK_CUDNN_FRONTEND_ERROR(set_fp8_desc(attributes.inputs,
+ SDPA_attributes::input_names::Descale_Q,
+ CUDNN_ATTR_OPERATION_SDPA_FWD_DESCALE_QDESC));
+ CHECK_CUDNN_FRONTEND_ERROR(set_fp8_desc(attributes.inputs,
+ SDPA_attributes::input_names::Descale_K,
+ CUDNN_ATTR_OPERATION_SDPA_FWD_DESCALE_KDESC));
+ CHECK_CUDNN_FRONTEND_ERROR(set_fp8_desc(attributes.inputs,
+ SDPA_attributes::input_names::Descale_V,
+ CUDNN_ATTR_OPERATION_SDPA_FWD_DESCALE_VDESC));
+ CHECK_CUDNN_FRONTEND_ERROR(set_fp8_desc(attributes.inputs,
+ SDPA_attributes::input_names::Descale_S,
+ CUDNN_ATTR_OPERATION_SDPA_FWD_DESCALE_SDESC));
+ CHECK_CUDNN_FRONTEND_ERROR(set_fp8_desc(
+ attributes.inputs, SDPA_attributes::input_names::Scale_S, CUDNN_ATTR_OPERATION_SDPA_FWD_SCALE_SDESC));
+ CHECK_CUDNN_FRONTEND_ERROR(set_fp8_desc(
+ attributes.inputs, SDPA_attributes::input_names::Scale_O, CUDNN_ATTR_OPERATION_SDPA_FWD_SCALE_ODESC));
+ CHECK_CUDNN_FRONTEND_ERROR(set_fp8_desc(
+ attributes.outputs, SDPA_attributes::output_names::Amax_S, CUDNN_ATTR_OPERATION_SDPA_FWD_AMAX_SDESC));
+ CHECK_CUDNN_FRONTEND_ERROR(set_fp8_desc(
+ attributes.outputs, SDPA_attributes::output_names::Amax_O, CUDNN_ATTR_OPERATION_SDPA_FWD_AMAX_ODESC));
+#else
+ return fp8_cudnn_ver_error;
+#endif // CUDNN_VERSION >= 93000
+ }
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(unified_sdpa_operation->get_backend_descriptor()));
+
+ raw_operations.push_back(unified_sdpa_operation);
+
+ auto const& non_virtual_uids = attributes.get_non_virtual_uids();
+ uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end());
+ return {error_code_t::OK, ""};
+#else
+ CUDNN_FRONTEND_UNUSED(uids_involved_in_operations);
+ CUDNN_FRONTEND_UNUSED(raw_operations);
+ CUDNN_FRONTEND_UNUSED(tensors);
+ return cudnn_ver_error;
+#endif // CUDNN_VERSION >= 91301
+ }
+
+ protected:
+ std::shared_ptr<Graph> subgraph; // Pre-softmax subgraph
+ std::shared_ptr<Tensor_attributes> subgraph_input;
+ std::shared_ptr<Tensor_attributes> subgraph_output;
+ std::shared_ptr<UnifiedSoftmaxNode> softmax_node; // Softmax descriptor for BE >= 9.21
+
+ void
+ init_subgraph() {
+ subgraph = std::make_shared<Graph>();
+ auto subgraph_node = std::static_pointer_cast<INode>(subgraph);
+ subgraph_node->context = context;
+ subgraph_input = std::make_shared<Tensor_attributes>();
+ subgraph_input->set_is_virtual(true);
+ subgraph_input->set_name(attributes.name + "::subgraph_input");
+ auto b = attributes.inputs[input_names::Q]->get_dim()[0];
+ auto h_q = attributes.inputs[input_names::Q]->get_dim()[1];
+ auto s_q = attributes.inputs[input_names::Q]->get_dim()[2];
+ auto s_kv = infer_s_kv();
+ subgraph_input->set_dim({b, h_q, s_q, s_kv});
+ subgraph_input->set_stride({h_q * s_q * s_kv, s_q * s_kv, s_kv, 1});
+ subgraph_output = subgraph_input;
+ }
+};
+
+} // namespace cudnn_frontend::graph
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/sdpa_fp8_bwd.h b/deps/cudnn-frontend/include/cudnn_frontend/node/sdpa_fp8_bwd.h
new file mode 100644
index 0000000..0cec5a8
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/node/sdpa_fp8_bwd.h
@@ -0,0 +1,1172 @@
+#pragma once
+
+#include "../../cudnn_frontend_Heuristics.h"
+#include "../../cudnn_frontend_Logging.h"
+
+#include "../graph_helpers.h"
+#include "../node_interface.h"
+
+#include "diagonal_band_mask.h"
+#include "matmul_fp8.h"
+#include "pointwise.h"
+#include "reduction.h"
+#include "softmax.h"
+#include "block_scale_dequantize.h"
+#include "scaled_dot_product_flash_attention.h" // For promote_1d_index_tensor_to_4d()
+
+namespace cudnn_frontend::graph {
+
+class SDPAFP8BackwardNode : public NodeCRTP<SDPAFP8BackwardNode> {
+ using input_names = SDPA_fp8_backward_attributes::input_names;
+ using output_names = SDPA_fp8_backward_attributes::output_names;
+
+ private:
+ mutable bool is_deterministic_algorithm_supported_on_blackwell = false; // Will be edited in pre_validate_node()
+
+ // Promote any 1-D seq_len / ragged-offset index tensors to the 4-D
+ // [n, 1, 1, 1] form the cuDNN backend requires (see promote_1d_index_tensor_to_4d).
+ void
+ promote_index_tensors_to_4d() {
+ // TODO: Handle CU_SEQ_LEN_Q and CU_SEQ_LEN_KV once fp8 bprop supports these.
+ for (auto& key : {input_names::SEQ_LEN_Q, input_names::SEQ_LEN_KV}) {
+ auto const it = attributes.inputs.find(key);
+ if (it != attributes.inputs.end()) {
+ promote_1d_index_tensor_to_4d(it->second);
+ }
+ }
+ for (auto& [key, value] : attributes.inputs) {
+ CUDNN_FRONTEND_UNUSED(key);
+ if (value != nullptr) {
+ promote_1d_index_tensor_to_4d(value->get_ragged_offset());
+ }
+ }
+ for (auto& [key, value] : attributes.outputs) {
+ CUDNN_FRONTEND_UNUSED(key);
+ if (value != nullptr) {
+ promote_1d_index_tensor_to_4d(value->get_ragged_offset());
+ }
+ }
+ }
+
+ public:
+ mutable SDPA_fp8_backward_attributes
+ attributes; // mutable to allow auto-routing to deterministic in pre_validate_node()
+
+ SDPAFP8BackwardNode(SDPA_fp8_backward_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP(context), attributes(std::move(attributes_)) {}
+
+ Type
+ getType() override final {
+ return Type::COMPOSITE;
+ }
+
+ // Helper function to detect MXFP8 (microscaling FP8) mode
+ // MXFP8 uses block-wise scale factors with E8M0 data type and F8_128x4 reordering
+ // When detected, we use block_scale_dequantize before matmuls instead of pointwise descale after
+ bool
+ is_mxfp8_scaling() const {
+ auto descale_q_it = attributes.inputs.find(input_names::Descale_Q);
+ if (descale_q_it == attributes.inputs.end() || descale_q_it->second == nullptr) {
+ return false;
+ }
+ auto const& descale_q = descale_q_it->second;
+ return (descale_q->get_data_type() == DataType_t::FP8_E8M0 &&
+ descale_q->get_reordering_type() == TensorReordering_t::F8_128x4);
+ }
+
+ error_t
+ pre_validate_node() const override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating SDPAFP8BackwardNode " << attributes.name);
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 90100,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "sdpa fp8 backward operation is only supported starting cudnn 9.1.0. Please "
+ "consider upgrading your current version.");
+
+ // Bug workaround for 9.10.0 (TE constraint)
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ detail::get_backend_version() == 91000,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "sdpa fp8 backward operation is not supported on cudnn 9.10.0 due to known bugs. "
+ "Please consider upgrading to 9.10.2 or newer.");
+
+ CHECK_CUDNN_FRONTEND_ERROR(context.populate_sm_version_from_device());
+ int32_t const sm_version = context.get_sm_version();
+ int32_t const prop_major = sm_version / 10;
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ prop_major < 9,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "sdpa fp8 forward operation is only supported on Hopper architecture and newer. Please "
+ "consider using a newer architecture.");
+
+ // check that Q, K, V, O, stats, dO, dQ, dK, dV tensors has been assigned
+ // check that dim and strides has been assigned and last stride is 1
+#define CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(port, port_map) \
+ { \
+ std::shared_ptr<Tensor_attributes> tensor_ptr = port_map.at(port); \
+ RETURN_CUDNN_FRONTEND_ERROR_IF(tensor_ptr->get_dim().size() != 4, \
+ error_code_t::ATTRIBUTE_NOT_SET, \
+ "The dim for " + std::string(#port) + " is invalid"); \
+ RETURN_CUDNN_FRONTEND_ERROR_IF(tensor_ptr->get_stride().size() != 4, \
+ error_code_t::ATTRIBUTE_NOT_SET, \
+ "The stride for " + std::string(#port) + " is invalid"); \
+ RETURN_CUDNN_FRONTEND_ERROR_IF( \
+ tensor_ptr->get_stride()[3] != 1, \
+ error_code_t::GRAPH_NOT_SUPPORTED, \
+ "The stride for the last dimension corresponding to the embedding size per head should be 1 for " + \
+ std::string(#port)); \
+ }
+
+ CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::Q, attributes.inputs);
+ CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::K, attributes.inputs);
+ CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::V, attributes.inputs);
+ CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::O, attributes.inputs);
+ CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::Stats, attributes.inputs);
+ CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::dO, attributes.inputs);
+ CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(output_names::dQ, attributes.outputs);
+ CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(output_names::dK, attributes.outputs);
+ CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(output_names::dV, attributes.outputs);
+
+#undef CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE
+
+ // validate backend limitations for the operation
+ // clang-format off
+ int64_t s_q = attributes.inputs.at(input_names::Q)->get_dim()[2];
+ int64_t s_kv = attributes.inputs.at(input_names::K)->get_dim()[2];
+ int64_t h_q = attributes.inputs.at(input_names::Q)->get_dim()[1];
+ int64_t h_k = attributes.inputs.at(input_names::K)->get_dim()[1];
+ int64_t h_v = attributes.inputs.at(input_names::V)->get_dim()[1];
+ int64_t d_qk = attributes.inputs.at(input_names::Q)->get_dim()[3];
+ int64_t d_v = attributes.inputs.at(input_names::V)->get_dim()[3];
+
+ auto const& dq_tensor = attributes.outputs.at(output_names::dQ);
+ auto const& dq_data_type = dq_tensor->get_data_type();
+ auto const& dk_tensor = attributes.outputs.at(output_names::dK);
+ auto const& dk_data_type = dk_tensor->get_data_type();
+ auto const& dv_tensor = attributes.outputs.at(output_names::dV);
+ auto const& dv_data_type = dv_tensor->get_data_type();
+
+ auto const& bias_mask = attributes.inputs.find(input_names::Bias);
+ bool const is_bias = (bias_mask != attributes.inputs.end() && bias_mask->second != nullptr);
+
+ auto const& dropout_mask = attributes.inputs.find(input_names::Dropout_mask);
+ bool const is_dropout_custom = (dropout_mask != attributes.inputs.end()) && (dropout_mask->second != nullptr);
+ bool const is_dropout = attributes.dropout_probability.has_value();
+ bool const is_ragged =
+ attributes.inputs.at(input_names::Q)->get_ragged_offset() ||
+ attributes.inputs.at(input_names::K)->get_ragged_offset() ||
+ attributes.inputs.at(input_names::V)->get_ragged_offset() ||
+ attributes.inputs.at(input_names::O)->get_ragged_offset() ||
+ attributes.inputs.at(input_names::Stats)->get_ragged_offset() ||
+ attributes.inputs.at(input_names::dO)->get_ragged_offset() ||
+ attributes.outputs.at(output_names::dQ)->get_ragged_offset() ||
+ attributes.outputs.at(output_names::dK)->get_ragged_offset() ||
+ attributes.outputs.at(output_names::dV)->get_ragged_offset();
+
+ // validation TODO:
+ // - validate stats has valid dims
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF((prop_major == 9) && is_ragged,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "sdpa fp8 backward with THD is not supported on Hopper architecture.");
+
+ // validate basic dimension requirements
+ if(prop_major >= 10) {
+ RETURN_CUDNN_FRONTEND_ERROR_IF(((d_qk > 128) || (d_qk % 16 != 0)) && !(d_qk == 192 && d_v == 128),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "hidden_dim d_qk shoud be less than or equal to 128 and hidden_dim d_qk should be multiple of 16 unless d_qk == 192 and d_v == 128");
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(((d_v > 128) || (d_v % 16 != 0)),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "hidden_dim d_v shoud be less than or equal to 128 and hidden_dim d_v should be multiple of 16");
+ }
+ else {
+ RETURN_CUDNN_FRONTEND_ERROR_IF((d_qk != 128) || (d_qk % 16 != 0) || (d_v != 128) || (d_v % 16 != 0),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "hidden_dim shoud be equal to 128 and hidden_dim should be multiple of 16");
+ }
+ RETURN_CUDNN_FRONTEND_ERROR_IF((h_q % h_k != 0) || (h_q % h_v != 0),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "For group-query attention, number of heads for key and query must be a factor of number of heads for query");
+
+ // validate options for attn_scale
+ auto const& attn_scale = attributes.inputs.find(input_names::Attn_scale);
+ bool const has_attn_scale = (attn_scale != attributes.inputs.end()) && (attn_scale->second != nullptr);
+ RETURN_CUDNN_FRONTEND_ERROR_IF(has_attn_scale && attributes.attn_scale_value.has_value(),
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "attn_scale with tensor and value cannot be set at the same time.");
+
+ // validate options for bias mask
+ RETURN_CUDNN_FRONTEND_ERROR_IF(is_bias && (bias_mask->second->get_data_type() == DataType_t::BOOLEAN),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Bias mask data type cannot be boolean");
+
+ // validate options for padding mask
+ auto const& seq_len_q = attributes.inputs.find(input_names::SEQ_LEN_Q);
+ bool const has_seq_len_q = (seq_len_q != attributes.inputs.end()) && (seq_len_q->second != nullptr);
+ auto const& seq_len_kv = attributes.inputs.find(input_names::SEQ_LEN_KV);
+ bool const has_seq_len_kv = (seq_len_kv != attributes.inputs.end()) && (seq_len_kv->second != nullptr);
+ RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.padding_mask && (!has_seq_len_q || !has_seq_len_kv),
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "Padding mask requires seq_len_q and seq_len_kv to be set.");
+ RETURN_CUDNN_FRONTEND_ERROR_IF((!attributes.padding_mask) && (has_seq_len_q || has_seq_len_kv),
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "seq_len_q and seq_len_kv needs to be set only if padding mask is enabled.");
+
+ // validate options for dropout mask
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ attributes.dropout_probability.has_value() && is_dropout_custom,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "Using both, custom dropout mask and internal-mask generation using dropout probability, is ill-formed.");
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ attributes.dropout_probability.has_value() && attributes.dropout_probability.value() == 1.0,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "Dropout probability cannot be 1 as corresponding scale wont be well formed.");
+
+ // Validate options for causal_mask_bottom_right
+ RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.has_causal_mask_bottom_right() && detail::get_backend_version() < 90700,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "For cuDNN version below 9.7.0, bottom right causal masking is not supported.");
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.has_causal_mask_bottom_right() && prop_major < 10,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "sdpa fp8 forward operation is only supported on Blackwell architecture and newer. Please "
+ "consider using a newer architecture.");
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.has_causal_mask_bottom_right() && s_q > s_kv,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Bottom right causal mask does not support s_q > s_kv. Please virtually slice the Q tensor and pass it as s_q == s_kv");
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.has_causal_mask_bottom_right() && (is_bias || is_dropout),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Bottom right causal mask is only supported with is_bias=False, is_dropout=False.");
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.has_causal_mask_bottom_right() && ((s_q % 64 != 0) || (s_kv % 64 != 0)),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Bottom right causal mask is only supported with s_q multiple of 64, and s_kv multiple of 64");
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ attributes.left_bound.has_value() && (!attributes.padding_mask) && s_q > s_kv,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Sliding window attention is only supported with max_s_q <= max_s_kv.");
+
+ // validate that datatype is set for the graph
+ RETURN_CUDNN_FRONTEND_ERROR_IF(context.get_intermediate_data_type() == DataType_t::NOT_SET,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "Intermediate tensor data type needs to be set as internal tensors require it.");
+
+ // Auto-route to deterministic algorithm for cases where non-deterministic is not supported on Blackwell:
+ // 1. MXFP8 backward - requires deterministic (STAGES=2)
+ // 2. FP8 backward with d_qk <= 192 and d_v <= 128 - requires deterministic
+ bool const is_mxfp8 = is_mxfp8_scaling();
+ bool const is_supported_dims = (d_qk <= 192) && (d_v <= 128);
+ if ((prop_major == 10) && !attributes.is_deterministic_algorithm) {
+ if (is_mxfp8) {
+ getLogger() << "[cudnn_frontend] INFO: MXFP8 SDPA backward detected on Blackwell - "
+ << "auto-routing to deterministic algorithm (non-deterministic not supported for MXFP8)"
+ << std::endl;
+ attributes.is_deterministic_algorithm = true;
+ } else if (is_supported_dims) {
+ getLogger() << "[cudnn_frontend] INFO: FP8 SDPA backward with d_qk=" << d_qk << ", d_v=" << d_v
+ << " detected on Blackwell - auto-routing to deterministic algorithm"
+ << std::endl;
+ attributes.is_deterministic_algorithm = true;
+ }
+ }
+
+ // validate options for deterministic algorithm
+ if (attributes.is_deterministic_algorithm && (prop_major == 10)) {
+ RETURN_CUDNN_FRONTEND_ERROR_IF((detail::get_backend_version() < 91900),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "FP8 deterministic algorithm (required for MXFP8 and d_qk=192/d_v=128) is not supported on Blackwell with cuDNN version below 9.19.0");
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(is_dropout,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "FP8 deterministic algorithm (required for MXFP8 and d_qk=192/d_v=128) is not supported on Blackwell when dropout is enabled");
+
+ is_deterministic_algorithm_supported_on_blackwell = true;
+ }
+
+ // if output data type is half or bfloat16 for any of dq, dk, dv, and version is below 9.13 or is not blackwell, return NOT_SUPPORTED
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ (dq_data_type == DataType_t::HALF || dq_data_type == DataType_t::BFLOAT16 ||
+ dk_data_type == DataType_t::HALF || dk_data_type == DataType_t::BFLOAT16 ||
+ dv_data_type == DataType_t::HALF || dv_data_type == DataType_t::BFLOAT16) &&
+ (detail::get_backend_version() < 91300 || prop_major < 10),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "sdpa fp8 backward with HALF/BFLOAT16 output is only supported on Blackwell architecture "
+ "with cuDNN version 9.13.0 and newer.");
+
+ // Validate MXFP8 scale factors if present
+ if (is_mxfp8_scaling()) {
+ // MXFP8 requires cuDNN 9.21.0 or later
+ RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 92100,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "MXFP8 SDPA requires cuDNN 9.21.0 or later");
+ int64_t const b = attributes.inputs.at(input_names::Q)->get_dim()[0];
+
+ // MXFP8 block size is fixed at 32
+ constexpr int64_t block_size = 32;
+ int64_t const d_qk_scale = (d_qk + block_size - 1) / block_size;
+ int64_t const d_v_scale = (d_v + block_size - 1) / block_size;
+ int64_t const s_q_scale = (s_q + block_size - 1) / block_size;
+ int64_t const s_kv_scale = (s_kv + block_size - 1) / block_size;
+
+ // Validate Descale_Q
+ auto const& descale_q = attributes.inputs.at(input_names::Descale_Q);
+ RETURN_CUDNN_FRONTEND_ERROR_IF(descale_q->get_data_type() != DataType_t::FP8_E8M0,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA requires Descale_Q to have FP8_E8M0 data type");
+ RETURN_CUDNN_FRONTEND_ERROR_IF(descale_q->get_reordering_type() != TensorReordering_t::F8_128x4,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA requires Descale_Q to have F8_128x4 reordering");
+
+ // Validate Descale_Q_T
+ auto const& descale_q_T = attributes.inputs.at(input_names::Descale_Q_T);
+ RETURN_CUDNN_FRONTEND_ERROR_IF(descale_q_T->get_data_type() != DataType_t::FP8_E8M0,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA requires Descale_Q_T to have FP8_E8M0 data type");
+ RETURN_CUDNN_FRONTEND_ERROR_IF(descale_q_T->get_reordering_type() != TensorReordering_t::F8_128x4,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA requires Descale_Q_T to have F8_128x4 reordering");
+
+ // Validate Descale_K
+ auto const& descale_k = attributes.inputs.at(input_names::Descale_K);
+ RETURN_CUDNN_FRONTEND_ERROR_IF(descale_k->get_data_type() != DataType_t::FP8_E8M0,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA requires Descale_K to have FP8_E8M0 data type");
+ RETURN_CUDNN_FRONTEND_ERROR_IF(descale_k->get_reordering_type() != TensorReordering_t::F8_128x4,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA requires Descale_K to have F8_128x4 reordering");
+
+ // Validate Descale_K_T
+ auto const& descale_k_T = attributes.inputs.at(input_names::Descale_K_T);
+ RETURN_CUDNN_FRONTEND_ERROR_IF(descale_k_T->get_data_type() != DataType_t::FP8_E8M0,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA requires Descale_K_T to have FP8_E8M0 data type");
+ RETURN_CUDNN_FRONTEND_ERROR_IF(descale_k_T->get_reordering_type() != TensorReordering_t::F8_128x4,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA requires Descale_K_T to have F8_128x4 reordering");
+
+ // Validate Descale_V
+ auto const& descale_v = attributes.inputs.at(input_names::Descale_V);
+ RETURN_CUDNN_FRONTEND_ERROR_IF(descale_v->get_data_type() != DataType_t::FP8_E8M0,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA requires Descale_V to have FP8_E8M0 data type");
+ RETURN_CUDNN_FRONTEND_ERROR_IF(descale_v->get_reordering_type() != TensorReordering_t::F8_128x4,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA requires Descale_V to have F8_128x4 reordering");
+
+ // Validate Descale_dO
+ auto const& descale_do = attributes.inputs.at(input_names::Descale_dO);
+ RETURN_CUDNN_FRONTEND_ERROR_IF(descale_do->get_data_type() != DataType_t::FP8_E8M0,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA requires Descale_dO to have FP8_E8M0 data type");
+ RETURN_CUDNN_FRONTEND_ERROR_IF(descale_do->get_reordering_type() != TensorReordering_t::F8_128x4,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA requires Descale_dO to have F8_128x4 reordering");
+
+ // Validate Descale_dO_T
+ auto const& descale_do_T = attributes.inputs.at(input_names::Descale_dO_T);
+ RETURN_CUDNN_FRONTEND_ERROR_IF(descale_do_T->get_data_type() != DataType_t::FP8_E8M0,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA requires Descale_dO_T to have FP8_E8M0 data type");
+ RETURN_CUDNN_FRONTEND_ERROR_IF(descale_do_T->get_reordering_type() != TensorReordering_t::F8_128x4,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA requires Descale_dO_T to have F8_128x4 reordering");
+
+ // Validate dimension consistency for SF_Q: [b, h_q, s_q_padded, d_scale_padded]
+ auto const& sf_q_dim = descale_q->get_dim();
+ RETURN_CUDNN_FRONTEND_ERROR_IF(sf_q_dim[0] != b || sf_q_dim[1] != h_q,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA: Descale_Q batch/head dimensions must match Q");
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ sf_q_dim[3] < d_qk_scale,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA: Descale_Q d_scale dimension too small (expected >= " + std::to_string(d_qk_scale) + ")");
+
+ // Validate dimension consistency for SF_Q_T: [b, h_q, s_q_scale_padded, d_padded]
+ auto const& sf_q_T_dim = descale_q_T->get_dim();
+ RETURN_CUDNN_FRONTEND_ERROR_IF(sf_q_T_dim[0] != b || sf_q_T_dim[1] != h_q,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA: Descale_Q_T batch/head dimensions must match Q");
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ sf_q_T_dim[2] < s_q_scale,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA: Descale_Q_T s_scale dimension too small (expected >= " + std::to_string(s_q_scale) + ")");
+
+ // Validate dimension consistency for SF_K: [b, h_k, s_kv_padded, d_qk_scale_padded]
+ auto const& sf_k_dim = descale_k->get_dim();
+ RETURN_CUDNN_FRONTEND_ERROR_IF(sf_k_dim[0] != b || sf_k_dim[1] != h_k,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA: Descale_K batch/head dimensions must match K");
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ sf_k_dim[3] < d_qk_scale,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA: Descale_K d_scale dimension too small (expected >= " + std::to_string(d_qk_scale) + ")");
+
+ // Validate dimension consistency for SF_K_T: [b, h_k, s_kv_scale_padded, d_qk_padded]
+ auto const& sf_k_T_dim = descale_k_T->get_dim();
+ RETURN_CUDNN_FRONTEND_ERROR_IF(sf_k_T_dim[0] != b || sf_k_T_dim[1] != h_k,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA: Descale_K_T batch/head dimensions must match K");
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ sf_k_T_dim[2] < s_kv_scale,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA: Descale_K_T s_scale dimension too small (expected >= " + std::to_string(s_kv_scale) + ")");
+
+ // Validate dimension consistency for SF_V: [b, h_v, s_kv_scale_padded, d_v_padded]
+ auto const& sf_v_dim = descale_v->get_dim();
+ RETURN_CUDNN_FRONTEND_ERROR_IF(sf_v_dim[0] != b || sf_v_dim[1] != h_v,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA: Descale_V batch/head dimensions must match V");
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ sf_v_dim[2] < s_kv_scale,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA: Descale_V s_scale dimension too small (expected >= " + std::to_string(s_kv_scale) + ")");
+
+ // Validate dimension consistency for SF_dO: [b, h_q, s_q_padded, d_v_scale_padded]
+ auto const& sf_do_dim = descale_do->get_dim();
+ RETURN_CUDNN_FRONTEND_ERROR_IF(sf_do_dim[0] != b || sf_do_dim[1] != h_q,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA: Descale_dO batch/head dimensions must match dO");
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ sf_do_dim[3] < d_v_scale,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA: Descale_dO d_scale dimension too small (expected >= " + std::to_string(d_v_scale) + ")");
+
+ // Validate dimension consistency for SF_dO_T: [b, h_q, s_q_scale_padded, d_scale_padded]
+ auto const& sf_do_T_dim = descale_do_T->get_dim();
+ RETURN_CUDNN_FRONTEND_ERROR_IF(sf_do_T_dim[0] != b || sf_do_T_dim[1] != h_q,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA: Descale_dO_T batch/head dimensions must match dO");
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ sf_do_T_dim[2] < s_q_scale,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "MXFP8 SDPA: Descale_dO_T s_scale dimension too small (expected >= " + std::to_string(s_q_scale) + ")");
+
+ // Log MXFP8 configuration for debugging
+ getLogger() << "[cudnn_frontend] INFO: MXFP8 SDPA configuration validated:" << std::endl
+ << " SF_Q dims: [" << sf_q_dim[0] << ", " << sf_q_dim[1] << ", " << sf_q_dim[2] << ", "
+ << sf_q_dim[3] << "]" << std::endl
+ << " SF_Q_T dims: [" << sf_q_T_dim[0] << ", " << sf_q_T_dim[1] << ", " << sf_q_T_dim[2] << ", "
+ << sf_q_T_dim[3] << "]" << std::endl
+ << " SF_K dims: [" << sf_k_dim[0] << ", " << sf_k_dim[1] << ", " << sf_k_dim[2] << ", "
+ << sf_k_dim[3] << "]" << std::endl
+ << " SF_K_T dims: [" << sf_k_T_dim[0] << ", " << sf_k_T_dim[1] << ", " << sf_k_T_dim[2] << ", "
+ << sf_k_T_dim[3] << "]" << std::endl
+ << " SF_V dims: [" << sf_v_dim[0] << ", " << sf_v_dim[1] << ", " << sf_v_dim[2] << ", "
+ << sf_v_dim[3] << "]" << std::endl
+ << " SF_dO dims: [" << sf_do_dim[0] << ", " << sf_do_dim[1] << ", " << sf_do_dim[2] << ", "
+ << sf_do_dim[3] << "]" << std::endl
+ << " SF_dO_T dims: [" << sf_do_T_dim[0] << ", " << sf_do_T_dim[1] << ", " << sf_do_T_dim[2] << ", "
+ << sf_do_T_dim[3] << "]" << std::endl
+ << " Expected d_qk_scale: " << d_qk_scale << ", d_v_scale: " << d_v_scale
+ << ", s_q_scale: " << s_q_scale << ", s_kv_scale: " << s_kv_scale << std::endl;
+ }
+
+ // validate options for sink token
+ auto const& sink_token = attributes.inputs.find(input_names::SINK_TOKEN);
+ bool const has_sink_token = (sink_token != attributes.inputs.end()) && (sink_token->second != nullptr);
+ auto const& dsink_token = attributes.outputs.find(output_names::DSINK_TOKEN);
+ bool const has_dsink_token = (dsink_token != attributes.outputs.end()) && (dsink_token->second != nullptr);
+ RETURN_CUDNN_FRONTEND_ERROR_IF(has_dsink_token && !has_sink_token,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "If dSink_token output is requested, sink_token input must also be set.");
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ infer_properties_node() override final {
+ // If a 1-D length/offset index tensor [n] is supplied, promote it to the 4-D
+ // [n, 1, 1, 1] form the cuDNN backend requires.
+ promote_index_tensors_to_4d();
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ expand_node() override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for Scaled_dot_product_flash_attention node "
+ << attributes.name);
+
+ attributes.fill_from_context(context);
+
+ // Gather dim to fill properties of virtual tensors
+ auto const& q_dim = attributes.inputs[input_names::Q]->get_dim();
+ auto b = q_dim[0];
+ auto h_q = q_dim[1];
+ auto s_q = q_dim[2];
+ auto d_qk = q_dim[3];
+ auto const& k_dim = attributes.inputs[input_names::K]->get_dim();
+ auto h_k = k_dim[1];
+ auto s_kv = k_dim[2];
+ auto const& v_dim = attributes.inputs[input_names::V]->get_dim();
+ auto h_v = v_dim[1];
+ auto d_v = v_dim[3];
+
+ bool const use_mxfp8 = is_mxfp8_scaling();
+ std::shared_ptr<Tensor_attributes> eff_Q, eff_K, eff_V, eff_dO;
+
+ // cuDNN frontend API attention requires Q, K, V where
+ // Q = {b, h_q, s_q, d_qk}
+ // K = {b, h_k, s_kv, d_qk}
+ // V = {b, h_v, s_kv, d_v}
+ // but cuDNN backend API attention requires Q, KT, VT
+ // Q = {b, h_q, s_q, d_qk}
+ // KT = {b, h_k, d_qk, s_kv}
+ // VT = {b, h_v, d_v, s_kv}
+ // So the code below maps the K->KT and V->VT
+ if (use_mxfp8) {
+ // MXFP8 2-kernel bprop: Each tensor is dequantized separately for each matmul it participates in.
+ // Q, K, dO each need two dequantizations with different scale factors.
+ // O and dO are used directly (as f16) for the dO*O computation without dequantization.
+
+ // K for BMM1 (Q@K.T): Transpose K and SF_K, then dequantize
+ std::vector<int64_t> kt_dim = attributes.inputs[input_names::K]->get_dim();
+ std::vector<int64_t> kt_stride = attributes.inputs[input_names::K]->get_stride();
+ std::swap(kt_dim[2], kt_dim[3]);
+ std::swap(kt_stride[2], kt_stride[3]);
+ attributes.inputs[input_names::K]->set_dim(kt_dim);
+ attributes.inputs[input_names::K]->set_stride(kt_stride);
+
+ auto& sf_k = attributes.inputs[input_names::Descale_K];
+ std::vector<int64_t> sf_k_dim = sf_k->get_dim();
+ std::vector<int64_t> sf_k_stride = sf_k->get_stride();
+ std::swap(sf_k_dim[2], sf_k_dim[3]);
+ std::swap(sf_k_stride[2], sf_k_stride[3]);
+ sf_k->set_dim(sf_k_dim);
+ sf_k->set_stride(sf_k_stride);
+
+ auto dequant_k_dqk_attrs = Block_scale_dequantize_attributes()
+ .set_name("DQ_K_dqk")
+ .set_block_size({1, 32});
+ auto k_dqk_dequant = std::make_shared<Tensor_attributes>();
+ k_dqk_dequant->set_is_virtual(true);
+ k_dqk_dequant->set_dim(kt_dim);
+ k_dqk_dequant->set_stride(kt_stride);
+ block_scale_dequantize(attributes.inputs[input_names::K], sf_k, dequant_k_dqk_attrs, k_dqk_dequant);
+ eff_K = k_dqk_dequant;
+
+ // V for dO@V.T: Transpose V and SF_V, then dequantize
+ std::vector<int64_t> vt_dim = attributes.inputs[input_names::V]->get_dim();
+ std::vector<int64_t> vt_stride = attributes.inputs[input_names::V]->get_stride();
+ std::swap(vt_dim[2], vt_dim[3]);
+ std::swap(vt_stride[2], vt_stride[3]);
+ attributes.inputs[input_names::V]->set_dim(vt_dim);
+ attributes.inputs[input_names::V]->set_stride(vt_stride);
+
+ auto& sf_v = attributes.inputs[input_names::Descale_V];
+ std::vector<int64_t> sf_v_dim = sf_v->get_dim();
+ std::vector<int64_t> sf_v_stride = sf_v->get_stride();
+ std::swap(sf_v_dim[2], sf_v_dim[3]);
+ std::swap(sf_v_stride[2], sf_v_stride[3]);
+ sf_v->set_dim(sf_v_dim);
+ sf_v->set_stride(sf_v_stride);
+
+ auto dequant_v_dv_attrs = Block_scale_dequantize_attributes()
+ .set_name("DQ_V_dv")
+ .set_block_size({1, 32});
+ auto v_dv_dequant = std::make_shared<Tensor_attributes>();
+ v_dv_dequant->set_is_virtual(true);
+ v_dv_dequant->set_dim(vt_dim);
+ v_dv_dequant->set_stride(vt_stride);
+ block_scale_dequantize(attributes.inputs[input_names::V], sf_v, dequant_v_dv_attrs, v_dv_dequant);
+ eff_V = v_dv_dequant;
+
+ // Q for BMM1 (Q@K.T): Dequantize Q with Descale_Q
+ auto dequant_q_dqk_attrs = Block_scale_dequantize_attributes()
+ .set_name("DQ_Q_dqk")
+ .set_block_size({1, 32});
+ auto q_dqk_dequant = std::make_shared<Tensor_attributes>();
+ q_dqk_dequant->set_is_virtual(true);
+ q_dqk_dequant->set_dim(attributes.inputs[input_names::Q]->get_dim());
+ q_dqk_dequant->set_stride(attributes.inputs[input_names::Q]->get_stride());
+ block_scale_dequantize(attributes.inputs[input_names::Q],
+ attributes.inputs[input_names::Descale_Q],
+ dequant_q_dqk_attrs,
+ q_dqk_dequant);
+ eff_Q = q_dqk_dequant;
+
+ // dO for dO@V.T: Dequantize dO with Descale_dO
+ auto dequant_dO_dv_attrs = Block_scale_dequantize_attributes()
+ .set_name("DQ_dO_dv")
+ .set_block_size({1, 32});
+ auto dO_dv_dequant = std::make_shared<Tensor_attributes>();
+ dO_dv_dequant->set_is_virtual(true);
+ dO_dv_dequant->set_dim(attributes.inputs[input_names::dO]->get_dim());
+ dO_dv_dequant->set_stride(attributes.inputs[input_names::dO]->get_stride());
+ block_scale_dequantize(attributes.inputs[input_names::dO],
+ attributes.inputs[input_names::Descale_dO],
+ dequant_dO_dv_attrs,
+ dO_dv_dequant);
+ eff_dO = dO_dv_dequant;
+
+ // Note: O and dO for the dO*O computation are used as raw f16 inputs (no MXFP8 dequant).
+ // The separate dequantizations for dO_seq, K_seq, Q_seq are done inline at their usage sites.
+ } else {
+ // Non-MXFP8
+ std::vector<int64_t> temp_vec;
+
+ temp_vec = attributes.inputs[input_names::K]->get_dim();
+ std::swap(temp_vec[2], temp_vec[3]);
+ attributes.inputs[input_names::K]->set_dim(temp_vec);
+
+ temp_vec = attributes.inputs[input_names::K]->get_stride();
+ std::swap(temp_vec[2], temp_vec[3]);
+ attributes.inputs[input_names::K]->set_stride(temp_vec);
+
+ temp_vec = attributes.inputs[input_names::V]->get_dim();
+ std::swap(temp_vec[2], temp_vec[3]);
+ attributes.inputs[input_names::V]->set_dim(temp_vec);
+
+ temp_vec = attributes.inputs[input_names::V]->get_stride();
+ std::swap(temp_vec[2], temp_vec[3]);
+ attributes.inputs[input_names::V]->set_stride(temp_vec);
+
+ eff_Q = attributes.inputs[input_names::Q];
+ eff_K = attributes.inputs[input_names::K];
+ eff_V = attributes.inputs[input_names::V];
+ eff_dO = attributes.inputs[input_names::dO];
+ }
+
+ std::shared_ptr<Tensor_attributes> rng_output;
+
+ auto mul_attributes = Pointwise_attributes().set_mode(PointwiseMode_t::MUL);
+
+ // if dropout_prob is used, then the node passes scale and scale inverse
+ // if dropout_mask is used, then the user passes scale and scale_inverse
+ bool is_dropout_prob = (attributes.dropout_probability.has_value());
+ bool is_dropout_mask = (attributes.inputs[input_names::Dropout_mask] != nullptr);
+ if (is_dropout_prob) {
+ float dropout_scale_value = 1.0f / (1.0f - attributes.dropout_probability.value());
+ float dropout_scale_inv_value = (1.0f - attributes.dropout_probability.value());
+
+ attributes.inputs[input_names::Dropout_scale] = std::make_shared<Tensor_attributes>(dropout_scale_value);
+ attributes.inputs[input_names::Dropout_scale_inv] =
+ std::make_shared<Tensor_attributes>(dropout_scale_inv_value);
+ }
+
+ // --------------RNG node--------------------
+
+ if (is_dropout_prob) {
+ rng_output = rng(attributes.inputs[input_names::Seed],
+ attributes.inputs[input_names::Offset],
+ Rng_attributes()
+ .set_name("rng")
+ .set_distribution(RngDistribution_t::BERNOULLI)
+ .set_bernoulli_probability(1.0f - attributes.dropout_probability.value()));
+ rng_output->set_dim({b, h_q, s_q, s_kv}).set_stride({h_q * s_q * s_kv, s_q * s_kv, s_kv, 1});
+ } else if (is_dropout_mask) {
+ rng_output = attributes.inputs[input_names::Dropout_mask];
+ }
+
+ //// dO * O
+ // For MXFP8: dO_f16 and O (f16) are used directly (no block dequantization needed)
+ // For non-MXFP8 (17-arg API): dO and O (FP8) are used (per-tensor descale applied after reduction)
+ mul_attributes.set_name("mul_dO_O");
+ auto last_output = use_mxfp8
+ ? pointwise(attributes.inputs[input_names::dO_f16], attributes.inputs[input_names::O], mul_attributes)
+ : pointwise(attributes.inputs[input_names::dO], attributes.inputs[input_names::O], mul_attributes);
+
+ // reduce(dO)
+ last_output =
+ reduction(last_output, Reduction_attributes().set_name("reduce_dO").set_mode(ReductionMode_t::ADD));
+ last_output->set_dim({b, h_q, s_q, 1}).set_stride({h_q * s_q, s_q, 1, 1});
+
+ // Descale dO (only for non-MXFP8 per-tensor scaling)
+ if (!use_mxfp8) {
+ mul_attributes.set_name("descale_dO");
+ last_output = pointwise(last_output, attributes.inputs.at(input_names::Descale_dO), mul_attributes);
+ last_output->set_dim({b, h_q, s_q, 1}).set_stride({h_q * s_q, s_q, 1, 1});
+ }
+
+ // Descale O (only for non-MXFP8 per-tensor scaling)
+ if (!use_mxfp8) {
+ mul_attributes.set_name("descale_O");
+ last_output = pointwise(last_output, attributes.inputs.at(input_names::Descale_O), mul_attributes);
+ }
+
+ // dSink_token computation: dSink = sum(exp(sink - stats) * D) over batch and sequence
+ // Note: The backend kernel applies the negative sign internally.
+ auto const& sink_token_it = attributes.inputs.find(input_names::SINK_TOKEN);
+ auto const& dsink_token_it = attributes.outputs.find(output_names::DSINK_TOKEN);
+ if (dsink_token_it != attributes.outputs.end() && dsink_token_it->second != nullptr &&
+ sink_token_it != attributes.inputs.end() && sink_token_it->second != nullptr) {
+ // sub_sink = sink - stats
+ auto sub_sink = pointwise(sink_token_it->second,
+ attributes.inputs[input_names::Stats],
+ Pointwise_attributes().set_name("sub_sink_stats").set_mode(PointwiseMode_t::SUB));
+ sub_sink->set_dim({b, h_q, s_q, 1}).set_stride({h_q * s_q, s_q, 1, 1});
+
+ // exp_sink = exp(sub_sink)
+ auto exp_sink =
+ pointwise(sub_sink, Pointwise_attributes().set_name("exp_sink").set_mode(PointwiseMode_t::EXP));
+ exp_sink->set_dim({b, h_q, s_q, 1}).set_stride({h_q * s_q, s_q, 1, 1});
+
+ // per_token_grad = exp_sink * last_output (D)
+ auto per_token_grad = pointwise(
+ exp_sink, last_output, Pointwise_attributes().set_name("mul_exp_sink_D").set_mode(PointwiseMode_t::MUL));
+ per_token_grad->set_dim({b, h_q, s_q, 1}).set_stride({h_q * s_q, s_q, 1, 1});
+
+ // dSink = reduce(per_token_grad) over batch and sequence dimensions
+ reduction(per_token_grad,
+ Reduction_attributes().set_name("reduce_dSink").set_mode(ReductionMode_t::ADD),
+ dsink_token_it->second);
+ }
+
+ // softmax_sum = last_output * dropout_scale
+ // Note: When sink_token is enabled and dropout is NOT present, we still need a MUL operation
+ // in the softmax path so the backend pattern matcher (fortAttentionBackwardRuntimeFusionEngine)
+ // can correctly distinguish between the dSink chain (MUL→REDUCE) and the softmax path (MUL→SUB).
+ // Without this, the fallback at lines 3786-3796 incorrectly identifies dSink MUL as the softmax path.
+ bool const has_sink_token = (sink_token_it != attributes.inputs.end() && sink_token_it->second != nullptr);
+ if (attributes.inputs[input_names::Dropout_scale_inv]) {
+ last_output = pointwise(last_output,
+ attributes.inputs[input_names::Dropout_scale_inv],
+ Pointwise_attributes().set_name("scale_dropout_inv").set_mode(PointwiseMode_t::MUL));
+ } else if (has_sink_token) {
+ // Add identity MUL (multiply by 1) to create MUL→SUB pattern for pattern matcher
+ auto one_tensor = std::make_shared<Tensor_attributes>(1.0f);
+ last_output = pointwise(last_output,
+ one_tensor,
+ Pointwise_attributes().set_name("identity_for_sink").set_mode(PointwiseMode_t::MUL));
+ last_output->set_dim({b, h_q, s_q, 1}).set_stride({h_q * s_q, s_q, 1, 1});
+ }
+ auto softmax_sum = last_output;
+
+ //// Q * K
+ auto bmm_Q_K_attributes = Matmul_attributes().set_name("bmm_Q_K")
+ .set_m_override(attributes.inputs[input_names::SEQ_LEN_Q])
+ .set_n_override(attributes.inputs[input_names::SEQ_LEN_KV]);
+ auto last_dV = matmul(eff_Q, eff_K, bmm_Q_K_attributes);
+
+ //// Optional Attn scale
+ // In case user provided a scalar value, do a fused scalar.
+ if (attributes.attn_scale_value.has_value()) {
+ attributes.inputs[input_names::Attn_scale] =
+ std::make_shared<Tensor_attributes>(attributes.attn_scale_value.value());
+ }
+
+ // If attn scale present, add a pointwise mul node
+ if (auto attn_scale_it = attributes.inputs.find(input_names::Attn_scale); attn_scale_it != attributes.inputs.end()) {
+ mul_attributes.set_name("attn_scale");
+ last_dV = pointwise(last_dV, attn_scale_it->second, mul_attributes);
+ }
+
+ // Descale Q (only for non-MXFP8 per-tensor scaling)
+ if (!use_mxfp8) {
+ mul_attributes.set_name("descale_q");
+ last_dV = pointwise(last_dV, attributes.inputs.at(input_names::Descale_Q), mul_attributes);
+ }
+
+ // Descale K (only for non-MXFP8 per-tensor scaling)
+ if (!use_mxfp8) {
+ mul_attributes.set_name("descale_k");
+ last_dV = pointwise(last_dV, attributes.inputs.at(input_names::Descale_K), mul_attributes);
+ }
+
+ // (optional) last_dV = last_dV + bias
+ if (auto bias_it = attributes.inputs.find(input_names::Bias); bias_it != attributes.inputs.end()) {
+ last_dV = pointwise(last_dV,
+ bias_it->second,
+ Pointwise_attributes().set_name("add_bias").set_mode(PointwiseMode_t::ADD));
+ }
+
+ // (optional) Apply padding mask
+ if (attributes.padding_mask) {
+ auto row_idx_output = pointwise(last_dV,
+ Pointwise_attributes()
+ .set_name("gen_row_idx_padding")
+ .set_mode(PointwiseMode_t::GEN_INDEX)
+ .set_axis(2)
+ .set_compute_data_type(DataType_t::INT32));
+ row_idx_output->set_data_type(DataType_t::INT32);
+
+ auto col_idx_output = pointwise(last_dV,
+ Pointwise_attributes()
+ .set_name("gen_col_idx_padding")
+ .set_mode(PointwiseMode_t::GEN_INDEX)
+ .set_axis(3)
+ .set_compute_data_type(DataType_t::INT32));
+ col_idx_output->set_data_type(DataType_t::INT32);
+
+ auto row_mask_output = pointwise(row_idx_output,
+ attributes.inputs[input_names::SEQ_LEN_Q],
+ Pointwise_attributes()
+ .set_name("lt_row_sq_padding")
+ .set_mode(PointwiseMode_t::CMP_LT)
+ .set_compute_data_type(DataType_t::BOOLEAN));
+ row_mask_output->set_data_type(DataType_t::BOOLEAN);
+
+ auto col_mask_output = pointwise(col_idx_output,
+ attributes.inputs[input_names::SEQ_LEN_KV],
+ Pointwise_attributes()
+ .set_name("lt_col_skv_padding")
+ .set_mode(PointwiseMode_t::CMP_LT)
+ .set_compute_data_type(DataType_t::BOOLEAN));
+ col_mask_output->set_data_type(DataType_t::BOOLEAN);
+
+ auto padding_mask_output = pointwise(row_mask_output,
+ col_mask_output,
+ Pointwise_attributes()
+ .set_name("and_row_col_padding")
+ .set_mode(PointwiseMode_t::LOGICAL_AND)
+ .set_compute_data_type(DataType_t::BOOLEAN));
+ padding_mask_output->set_data_type(DataType_t::BOOLEAN);
+
+ // Use a smaller value of neg infinity so that the softmax stats for rows that are fully padded dont
+ // go towards NaNs/Infs when multipled by the numerous scale/descale
+ auto negative_inf_padding = std::make_shared<Tensor_attributes>(attn::score_modifiers::get_negative_inf_value());
+
+ last_dV =
+ pointwise(last_dV,
+ negative_inf_padding,
+ padding_mask_output,
+ Pointwise_attributes().set_name("select_padding").set_mode(PointwiseMode_t::BINARY_SELECT));
+ }
+
+ // Apply (bottom-right) causal masking (with right bound) and/or set the left bound
+ if (attributes.left_bound.has_value() || attributes.right_bound.has_value()) {
+ auto graph_ = std::make_shared<Graph>();
+ std::shared_ptr<INode> node_ = std::static_pointer_cast<INode>(graph_);
+ node_->context = this->context;
+
+ auto s_kv_ptr = attributes.inputs.find(input_names::SEQ_LEN_KV) != attributes.inputs.end()
+ ? attributes.inputs[input_names::SEQ_LEN_KV]
+ : nullptr;
+ auto s_q_ptr = attributes.inputs.find(input_names::SEQ_LEN_Q) != attributes.inputs.end()
+ ? attributes.inputs[input_names::SEQ_LEN_Q]
+ : nullptr;
+
+ last_dV = attn::score_modifiers::sliding_window_mask(graph_,
+ last_dV,
+ attributes.diagonal_alignment,
+ attributes.left_bound,
+ attributes.right_bound,
+ s_q,
+ s_kv,
+ s_q_ptr,
+ s_kv_ptr,
+ /*cu_s_q_ptr=*/nullptr,
+ /*cu_s_kv_ptr=*/nullptr);
+ sub_nodes.emplace_back(std::move(node_));
+ }
+
+ //// Apply Softmax
+ // last_dV = last_dV - stats
+ last_dV = pointwise(last_dV,
+ attributes.inputs[input_names::Stats],
+ Pointwise_attributes().set_name("sub_dV_Stats").set_mode(PointwiseMode_t::SUB));
+
+ // last_dV = exp(last_dV)
+ last_dV = pointwise(last_dV, Pointwise_attributes().set_name("exp_dV").set_mode(PointwiseMode_t::EXP));
+ auto exp_S = last_dV;
+
+ // (optional) last_dV = last_dV * dropout rng_output
+ if (is_dropout_prob || is_dropout_mask) {
+ last_dV =
+ pointwise(last_dV,
+ rng_output,
+ Pointwise_attributes().set_name("mul_p_dropout_mask").set_mode(PointwiseMode_t::MUL));
+ }
+
+ // (optional) last_dV = last_dV * dropout_scale
+ if (attributes.inputs[input_names::Dropout_scale]) {
+ last_dV =
+ pointwise(last_dV,
+ attributes.inputs[input_names::Dropout_scale],
+ Pointwise_attributes().set_name("mul_dS_dropout_scale").set_mode(PointwiseMode_t::MUL));
+ }
+
+ // Scale S (only for non-MXFP8 per-tensor scaling; MXFP8 keeps P in fp32)
+ if (!use_mxfp8) {
+ mul_attributes.set_name("scale_S");
+ last_dV = pointwise(last_dV, attributes.inputs.at(input_names::Scale_S), mul_attributes);
+ last_dV->set_data_type(attributes.inputs.at(input_names::Q)->get_data_type());
+ }
+
+ // Reshape P
+ last_dV = reshape(last_dV, Reshape_attributes().set_name("S_transpose"));
+ last_dV->set_name("S_T").set_dim({b, h_q, s_kv, s_q}).set_stride({h_q * s_q * s_kv, s_q * s_kv, 1, s_kv});
+ if (!use_mxfp8) {
+ last_dV->set_data_type(attributes.inputs[input_names::Q]->get_data_type());
+ }
+
+ //// P.T @ dO -> dV
+ if (use_mxfp8) {
+ // MXFP8: Dequantize dO_T with Descale_dO_T for P.T@dO
+ auto dequant_dO_seq_attrs = Block_scale_dequantize_attributes()
+ .set_name("DQ_dO_seq")
+ .set_block_size({1, 32});
+ auto dO_seq_dequant = std::make_shared<Tensor_attributes>();
+ dO_seq_dequant->set_is_virtual(true);
+ dO_seq_dequant->set_dim(attributes.inputs[input_names::dO_T]->get_dim());
+ dO_seq_dequant->set_stride(attributes.inputs[input_names::dO_T]->get_stride());
+ block_scale_dequantize(attributes.inputs[input_names::dO_T],
+ attributes.inputs[input_names::Descale_dO_T],
+ dequant_dO_seq_attrs,
+ dO_seq_dequant);
+
+ auto bmm_S_T_dO_attributes = Matmul_attributes().set_name("bmm_S_T_dO")
+ .set_m_override(attributes.inputs[input_names::SEQ_LEN_KV])
+ .set_k_override(attributes.inputs[input_names::SEQ_LEN_Q]);
+ if (h_q == h_v) {
+ matmul(last_dV, dO_seq_dequant, bmm_S_T_dO_attributes, attributes.outputs[output_names::dV]);
+ } else {
+ auto dV_fullhead = matmul(last_dV, dO_seq_dequant, bmm_S_T_dO_attributes);
+ dV_fullhead->set_dim({b, h_q, s_kv, d_v});
+ dV_fullhead->set_stride({h_q * s_kv * d_v, s_kv * d_v, d_v, 1});
+ dV_fullhead->set_data_type(DataType_t::FLOAT);
+ reduction(dV_fullhead,
+ Reduction_attributes().set_name("red_dV_head").set_mode(ReductionMode_t::ADD),
+ attributes.outputs[output_names::dV]);
+ }
+
+ auto const& amax_dv = attributes.outputs.at(output_names::Amax_dV);
+ if (amax_dv != nullptr) {
+ auto amax_dv_attributes =
+ Reduction_attributes().set_name("amax_dV").set_mode(ReductionMode_t::AMAX);
+ reduction(attributes.outputs[output_names::dV], amax_dv_attributes, amax_dv);
+ }
+ } else {
+ matmul_fp8(last_dV,
+ attributes.inputs[input_names::dO],
+ attributes.inputs[input_names::Descale_S],
+ attributes.inputs[input_names::Descale_dO],
+ attributes.inputs[input_names::Scale_dV],
+ Matmul_fp8_attributes().set_name("bmm_S_T_dO")
+ .set_m_override(attributes.inputs[input_names::SEQ_LEN_KV])
+ .set_k_override(attributes.inputs[input_names::SEQ_LEN_Q]),
+ attributes.outputs[output_names::dV],
+ attributes.outputs[output_names::Amax_dV]);
+ }
+
+ //// dO * V_T
+ auto bmm_dO_V_T_attributes = Matmul_attributes().set_name("bmm_dO_V_T")
+ .set_m_override(attributes.inputs[input_names::SEQ_LEN_Q])
+ .set_n_override(attributes.inputs[input_names::SEQ_LEN_KV]);
+ last_output = matmul(eff_dO, eff_V, bmm_dO_V_T_attributes);
+
+ // Descale dO (only for non-MXFP8 per-tensor scaling)
+ if (!use_mxfp8) {
+ mul_attributes.set_name("descale_dO");
+ last_output = pointwise(last_output, attributes.inputs.at(input_names::Descale_dO), mul_attributes);
+ }
+
+ // Descale V (only for non-MXFP8 per-tensor scaling)
+ if (!use_mxfp8) {
+ mul_attributes.set_name("descale_V");
+ last_output = pointwise(last_output, attributes.inputs.at(input_names::Descale_V), mul_attributes);
+ }
+
+ // dP = last_output - softmax_sum
+ auto dP = pointwise(last_output,
+ softmax_sum,
+ Pointwise_attributes().set_name("sub_dP_softmax_sum").set_mode(PointwiseMode_t::SUB));
+
+ // dP = dP * exp_S
+ mul_attributes.set_name("mul_dP_exp_S");
+ dP = pointwise(dP, exp_S, mul_attributes);
+
+ // (optional) dP = dP * dropout_scale
+ if (attributes.inputs[input_names::Dropout_scale]) {
+ dP =
+ pointwise(dP,
+ attributes.inputs[input_names::Dropout_scale],
+ Pointwise_attributes().set_name("mul_dS_dropout_scale").set_mode(PointwiseMode_t::MUL));
+ }
+
+ // if (attributes.outputs[output_names::dBias]) {
+ // reduction(dP,
+ // Reduction_attributes().set_name("red_dP_dBias").set_mode(ReductionMode_t::ADD),
+ // attributes.outputs[output_names::dBias]);
+ // }
+
+ // (optional) dP = dP * attn_scale
+ if (auto attn_scale_it = attributes.inputs.find(input_names::Attn_scale); attn_scale_it != attributes.inputs.end()) {
+ mul_attributes.set_name("mul_dS_attn_scale");
+ dP = pointwise(dP, attn_scale_it->second, mul_attributes);
+ }
+
+ // Amax dP and Scale dP (only for non-MXFP8 per-tensor scaling; MXFP8 keeps dS in fp32)
+ if (!use_mxfp8) {
+ auto amax_attributes = Reduction_attributes().set_name("amax_dP").set_mode(ReductionMode_t::AMAX);
+ reduction(dP, amax_attributes, attributes.outputs.at(output_names::Amax_dP));
+
+ mul_attributes.set_name("scale_dP");
+ dP = pointwise(dP, attributes.inputs.at(input_names::Scale_dP), mul_attributes);
+ dP->set_data_type(attributes.inputs.at(input_names::dO)->get_data_type());
+ }
+
+ //// dS @ K -> dQ
+ if (use_mxfp8) {
+ // MXFP8: Dequantize K_T with Descale_K_T (seq-dimension scaling) for dS@K -> dQ
+ auto dequant_K_seq_attrs = Block_scale_dequantize_attributes()
+ .set_name("DQ_K_seq")
+ .set_block_size({1, 32});
+ auto K_seq_dequant = std::make_shared<Tensor_attributes>();
+ K_seq_dequant->set_is_virtual(true);
+ K_seq_dequant->set_dim(attributes.inputs[input_names::K_T]->get_dim());
+ K_seq_dequant->set_stride(attributes.inputs[input_names::K_T]->get_stride());
+
+ int64_t s_kv_scale_padded = ((s_kv + 127) / 128) * 4;
+ auto& sf_k_T = attributes.inputs[input_names::Descale_K_T];
+ sf_k_T->set_stride({h_k * s_kv_scale_padded * 128, s_kv_scale_padded * 128, b * h_k * s_kv_scale_padded * 128, 1});
+
+ block_scale_dequantize(attributes.inputs[input_names::K_T],
+ attributes.inputs[input_names::Descale_K_T],
+ dequant_K_seq_attrs,
+ K_seq_dequant);
+
+ auto bmm_dS_K_attributes = Matmul_attributes().set_name("bmm_dS_K")
+ .set_m_override(attributes.inputs[input_names::SEQ_LEN_Q])
+ .set_k_override(attributes.inputs[input_names::SEQ_LEN_KV]);
+ matmul(dP, K_seq_dequant, bmm_dS_K_attributes, attributes.outputs[output_names::dQ]);
+
+ auto const& amax_dq = attributes.outputs.at(output_names::Amax_dQ);
+ if (amax_dq != nullptr) {
+ auto amax_dq_attributes =
+ Reduction_attributes().set_name("amax_dQ").set_mode(ReductionMode_t::AMAX);
+ reduction(attributes.outputs[output_names::dQ], amax_dq_attributes, amax_dq);
+ }
+ } else {
+ auto const& kt_dim = attributes.inputs[input_names::K]->get_dim();
+ auto const& kt_stride = attributes.inputs[input_names::K]->get_stride();
+
+ auto K = reshape(attributes.inputs[input_names::K], Reshape_attributes().set_name("reshape_K"));
+ K->set_dim({kt_dim[0], kt_dim[1], kt_dim[3], kt_dim[2]})
+ .set_stride({kt_stride[0], kt_stride[1], kt_stride[3], kt_stride[2]});
+
+ auto bmm_dP_K_attributes = Matmul_fp8_attributes().set_name("bmm_dP_K")
+ .set_m_override(attributes.inputs[input_names::SEQ_LEN_Q])
+ .set_k_override(attributes.inputs[input_names::SEQ_LEN_KV]);
+ matmul_fp8(dP,
+ K,
+ attributes.inputs[input_names::Descale_dP],
+ attributes.inputs[input_names::Descale_K],
+ attributes.inputs[input_names::Scale_dQ],
+ bmm_dP_K_attributes,
+ attributes.outputs[output_names::dQ],
+ attributes.outputs[output_names::Amax_dQ]);
+ }
+
+ //// dS.T * Q (transpose dS -> dS_reshape)
+ auto dP_T_attributes = Reshape_attributes().set_name("dP_T");
+ auto dP_T = reshape(dP, dP_T_attributes);
+ if (!use_mxfp8) {
+ dP_T->set_data_type(attributes.inputs.at(input_names::dO)->get_data_type());
+ }
+ dP_T->set_name("dP_T").set_dim({b, h_q, s_kv, s_q}).set_stride({h_q * s_q * s_kv, s_q * s_kv, 1, s_kv});
+
+ if (use_mxfp8) {
+ // MXFP8: Dequantize Q_T with Descale_Q_T (seq-dimension scaling) for dS.T@Q -> dK
+ auto dequant_Q_seq_attrs = Block_scale_dequantize_attributes()
+ .set_name("DQ_Q_seq")
+ .set_block_size({1, 32});
+ auto Q_seq_dequant = std::make_shared<Tensor_attributes>();
+ Q_seq_dequant->set_is_virtual(true);
+ Q_seq_dequant->set_dim(attributes.inputs[input_names::Q_T]->get_dim());
+ Q_seq_dequant->set_stride(attributes.inputs[input_names::Q_T]->get_stride());
+
+ int64_t s_q_scale_padded = ((s_q + 127) / 128) * 4;
+ auto& sf_q_T = attributes.inputs[input_names::Descale_Q_T];
+ sf_q_T->set_stride({h_q * s_q_scale_padded * 128, s_q_scale_padded * 128, b * h_q * s_q_scale_padded * 128, 1});
+
+ block_scale_dequantize(attributes.inputs[input_names::Q_T],
+ attributes.inputs[input_names::Descale_Q_T],
+ dequant_Q_seq_attrs,
+ Q_seq_dequant);
+
+ auto bmm_dS_T_Q_attributes = Matmul_attributes().set_name("bmm_dS_T_Q")
+ .set_m_override(attributes.inputs[input_names::SEQ_LEN_KV])
+ .set_k_override(attributes.inputs[input_names::SEQ_LEN_Q]);
+ if (h_q == h_k) {
+ matmul(dP_T, Q_seq_dequant, bmm_dS_T_Q_attributes, attributes.outputs[output_names::dK]);
+ } else {
+ auto dK_fullhead = matmul(dP_T, Q_seq_dequant, bmm_dS_T_Q_attributes);
+ dK_fullhead->set_dim({b, h_q, s_kv, d_qk});
+ dK_fullhead->set_stride({h_q * s_kv * d_qk, s_kv * d_qk, d_qk, 1});
+ dK_fullhead->set_data_type(DataType_t::FLOAT);
+ reduction(dK_fullhead,
+ Reduction_attributes().set_name("red_dK_head").set_mode(ReductionMode_t::ADD),
+ attributes.outputs[output_names::dK]);
+ }
+
+ auto const& amax_dk = attributes.outputs.at(output_names::Amax_dK);
+ if (amax_dk != nullptr) {
+ auto amax_dk_attributes =
+ Reduction_attributes().set_name("amax_dK").set_mode(ReductionMode_t::AMAX);
+ reduction(attributes.outputs[output_names::dK], amax_dk_attributes, amax_dk);
+ }
+ } else {
+ auto bmm_dP_T_Q_attributes = Matmul_fp8_attributes().set_name("bmm_dP_T_Q")
+ .set_m_override(attributes.inputs[input_names::SEQ_LEN_KV])
+ .set_k_override(attributes.inputs[input_names::SEQ_LEN_Q]);
+ matmul_fp8(dP_T,
+ attributes.inputs[input_names::Q],
+ attributes.inputs[input_names::Descale_dP],
+ attributes.inputs[input_names::Descale_Q],
+ attributes.inputs[input_names::Scale_dK],
+ bmm_dP_T_Q_attributes,
+ attributes.outputs[output_names::dK],
+ attributes.outputs[output_names::Amax_dK]);
+ }
+
+ return {error_code_t::OK, ""};
+ }
+
+ std::pair<int64_t, std::unordered_map<KnobType_t, int64_t>>
+ override_heuristics_query() const {
+ int32_t const sm_version = context.get_sm_version();
+ bool const use_new_knobs = detail::get_backend_version() >= 92300;
+ // {128,128} bprop: tileM=3, tileN=2, kernelCfg=2(bprop warp), streamK=0, cgaM=0
+ if (sm_version > 103 && is_deterministic_algorithm_supported_on_blackwell) {
+ if (use_new_knobs) {
+ return {17,
+ {{KnobType_t::TILE_M, 3},
+ {KnobType_t::TILE_N, 2},
+ {KnobType_t::KERNEL_CFG, 2},
+ {KnobType_t::STREAM_K, 0},
+ {KnobType_t::TILE_CGA_M, 0},
+ {KnobType_t::STAGES, 2}}};
+ } else {
+ return {17, {{KnobType_t::KERNEL_CFG, 31}, {KnobType_t::STAGES, 2}}};
+ }
+ } else if (is_deterministic_algorithm_supported_on_blackwell) {
+ if (use_new_knobs) {
+ return {5,
+ {{KnobType_t::TILE_M, 3},
+ {KnobType_t::TILE_N, 2},
+ {KnobType_t::KERNEL_CFG, 2},
+ {KnobType_t::STREAM_K, 0},
+ {KnobType_t::TILE_CGA_M, 0},
+ {KnobType_t::STAGES, 2}}};
+ } else {
+ return {5, {{KnobType_t::KERNEL_CFG, 31}, {KnobType_t::STAGES, 2}}};
+ }
+ } else {
+ return {-1, {}};
+ }
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ j["is_mxfp8"] = is_mxfp8_scaling();
+ if (auto const rescale_threshold = get_rescale_threshold_from_env(); rescale_threshold.has_value()) {
+ j["rescale_threshold"] = rescale_threshold.value();
+ }
+ if (is_mxfp8_scaling()) {
+ j.update(R"({"tag": "SDPA_MXFP8_BWD"})"_json);
+ } else {
+ j.update(R"({"tag": "SDPA_FP8_BWD"})"_json);
+ }
+ }
+#endif
+};
+
+} // namespace cudnn_frontend::graph
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/sdpa_support_surface.h b/deps/cudnn-frontend/include/cudnn_frontend/node/sdpa_support_surface.h
new file mode 100644
index 0000000..6b73f73
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/node/sdpa_support_surface.h
@@ -0,0 +1,553 @@
+#pragma once
+
+#include <cstdlib>
+#include <unordered_set>
+
+#include "../../cudnn_frontend_Heuristics.h"
+#include "../../cudnn_frontend_Logging.h"
+#include "../graph_helpers.h"
+#include "../node_interface.h"
+
+namespace cudnn_frontend::graph {
+
+inline error_t
+SDPA_attributes::validate_sdpa_support_surface(const detail::Context& context,
+ int64_t s_kv,
+ bool is_paged_k,
+ bool is_paged_v) const {
+ // Extract dimensions from tensors
+ int64_t s_q = inputs.at(SDPA_attributes::input_names::Q)->get_dim()[2];
+ // s_kv is passed in from the caller
+ int64_t h_q = inputs.at(SDPA_attributes::input_names::Q)->get_dim()[1];
+ int64_t h_k = inputs.at(SDPA_attributes::input_names::K)->get_dim()[1];
+ int64_t h_v = inputs.at(SDPA_attributes::input_names::V)->get_dim()[1];
+ int64_t d_qk = inputs.at(SDPA_attributes::input_names::Q)->get_dim()[3];
+ int64_t d_v = inputs.at(SDPA_attributes::input_names::V)->get_dim()[3];
+
+ bool const is_ragged = inputs.at(SDPA_attributes::input_names::Q)->get_ragged_offset() ||
+ inputs.at(SDPA_attributes::input_names::K)->get_ragged_offset() ||
+ inputs.at(SDPA_attributes::input_names::V)->get_ragged_offset() ||
+ outputs.at(SDPA_attributes::output_names::O)->get_ragged_offset();
+
+ auto const& output_tensor = outputs.at(SDPA_attributes::output_names::O);
+ auto const& output_data_type = output_tensor->get_data_type();
+
+ auto const& bias_mask = inputs.find(SDPA_attributes::input_names::Bias);
+ bool const is_bias = (bias_mask != inputs.end() && bias_mask->second != nullptr);
+
+ auto const& dropout_mask = inputs.find(SDPA_attributes::input_names::Dropout_mask);
+ bool const is_dropout_custom = (dropout_mask != inputs.end()) && (dropout_mask->second != nullptr);
+ bool const is_dropout = dropout_probability.has_value() || is_dropout_custom;
+
+ bool const is_paged = is_paged_k || is_paged_v;
+
+ auto const& rng_tensor = outputs.find(SDPA_attributes::output_names::RNG_DUMP);
+ bool const is_rng = (rng_tensor != outputs.end() && rng_tensor->second != nullptr);
+
+ bool const max_seq_kv_explicit = max_seq_len_kv.has_value();
+
+ auto const& attn_scale = inputs.find(SDPA_attributes::input_names::Attn_scale);
+ bool const has_attn_scale = (attn_scale != inputs.end()) && (attn_scale->second != nullptr);
+
+ auto const& seq_len_q = inputs.find(SDPA_attributes::input_names::SEQ_LEN_Q);
+ bool const has_seq_len_q = (seq_len_q != inputs.end()) && (seq_len_q->second != nullptr);
+ auto const& seq_len_kv = inputs.find(SDPA_attributes::input_names::SEQ_LEN_KV);
+ bool const has_seq_len_kv = (seq_len_kv != inputs.end()) && (seq_len_kv->second != nullptr);
+
+ auto const& cu_seq_len_q = inputs.find(SDPA_attributes::input_names::CU_SEQ_LEN_Q);
+ bool const has_cu_seq_len_q = (cu_seq_len_q != inputs.end()) && (cu_seq_len_q->second != nullptr);
+ auto const& cu_seq_len_kv = inputs.find(SDPA_attributes::input_names::CU_SEQ_LEN_KV);
+ bool const has_cu_seq_len_kv = (cu_seq_len_kv != inputs.end()) && (cu_seq_len_kv->second != nullptr);
+
+ // validation TODO:
+ // - validate stats has valid dims
+
+ // Get device SM version from context
+ CHECK_CUDNN_FRONTEND_ERROR(context.populate_sm_version_from_device());
+ int32_t const sm_version = context.get_sm_version();
+ int32_t const prop_major = sm_version / 10;
+
+ // Common FP16 and FP8 validation
+ // validate basic dimension requirements
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ (h_q % h_k != 0) || (h_q % h_v != 0),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "For group-query attention, number of heads for key and query must be a factor of number of heads for query");
+
+ // validate options for attn_scale
+ RETURN_CUDNN_FRONTEND_ERROR_IF(has_attn_scale && attn_scale_value.has_value(),
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "attn_scale with tensor and value cannot be set at the same time.");
+
+ // validate options for bias mask
+ RETURN_CUDNN_FRONTEND_ERROR_IF(is_bias && (bias_mask->second->get_data_type() == DataType_t::BOOLEAN),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Bias mask data type cannot be boolean");
+
+ // validate options for padding mask: padding requires per-sequence length tensors,
+ // either as (SEQ_LEN_Q + SEQ_LEN_KV) or as (CU_SEQ_LEN_Q + CU_SEQ_LEN_KV).
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ padding_mask && (!has_seq_len_q || !has_seq_len_kv) && (!has_cu_seq_len_q || !has_cu_seq_len_kv),
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "Padding mask requires seq_len_q/seq_len_kv (or cu_seq_len_q/cu_seq_len_kv) to be set.");
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ (!padding_mask && !attention_score_modifier) &&
+ (has_seq_len_q || has_seq_len_kv || has_cu_seq_len_q || has_cu_seq_len_kv),
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "seq_len_q/seq_len_kv (or cu_seq_len_q/cu_seq_len_kv) needs to be set only if padding mask is enabled.");
+
+ // Cumulative sequence length tensors must be set together, and are mutually
+ // exclusive with the (per-batch) seq_len tensors.
+ RETURN_CUDNN_FRONTEND_ERROR_IF(has_cu_seq_len_q != has_cu_seq_len_kv,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "cu_seq_len_q and cu_seq_len_kv must both be set or both unset.");
+ RETURN_CUDNN_FRONTEND_ERROR_IF((has_cu_seq_len_q || has_cu_seq_len_kv) && (has_seq_len_q || has_seq_len_kv),
+ error_code_t::INVALID_VALUE,
+ "Cannot specify both seq_len tensors and cu_seq_len tensors.");
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(is_ragged && ((padding_mask == false) && (attention_score_modifier == nullptr)),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Ragged offsets are only supported with padding mask.");
+
+ // validate options for dropout mask
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ dropout_probability.has_value() && is_dropout_custom,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "Using both, custom dropout mask and internal-mask generation using dropout probability, is ill-formed.");
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(dropout_probability.has_value() && dropout_probability.value() == 1.0,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "Dropout probability cannot be 1 as corresponding scale wont be well formed.");
+
+ // validate options for causal mask and bottom right causal mask
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ has_causal_mask_bottom_right() && (!padding_mask) && s_q > s_kv,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Bottom right causal mask does not support max_s_q > max_s_kv. Please virtually slice the Q tensor and pass it "
+ "as max_s_q == max_s_kv");
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ has_causal_mask_bottom_right() && (is_bias || alibi_mask || is_dropout),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Bottom right causal mask is only supported with is_bias=False, is_alibi=False, is_dropout=False.");
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(has_causal_mask_bottom_right() && (detail::get_backend_version() < 90600) &&
+ ((s_q % 64 != 0) || (s_kv % 64 != 0)),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Bottom right causal mask is only supported with s_q multiple of 64, and s_kv "
+ "multiple of 64, for cudnn version below 9.6.0");
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 90600 && left_bound.has_value() &&
+ has_causal_mask_bottom_right() && s_q != s_kv,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Sliding window attention with bottom right causal mask requires s_q == s_kv "
+ "for cuDNN version below 9.6.0");
+
+ // validate that datatype is set for the graph
+ RETURN_CUDNN_FRONTEND_ERROR_IF(context.get_intermediate_data_type() == DataType_t::NOT_SET,
+ error_code_t::ATTRIBUTE_NOT_SET,
+ "Intermediate tensor data type needs to be set as internal tensors require it.");
+
+ if (mma_core_mode == DataType_t::FP8_E4M3 || mma_core_mode == DataType_t::FP8_E5M2) {
+ // FP8 specific validation
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF((prop_major == 12) && is_ragged,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "sdpa fp8 with THD not supported for sm120 yet.");
+
+ // version specific validation
+ RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 90100,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "sdpa fp8 forward operation is only supported starting cudnn 9.1.0. Please "
+ "consider upgrading your current version.");
+ RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() == 91000,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "sdpa fp8 forward operation is not supported on cudnn 9.10.0. Please "
+ "consider upgrading your current version.");
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ prop_major < 9,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "sdpa fp8 forward operation is only supported on Hopper architecture and newer. Please "
+ "consider using a newer architecture.");
+
+ // FP8 does not support bias (TE constraint)
+ RETURN_CUDNN_FRONTEND_ERROR_IF(is_bias, error_code_t::GRAPH_NOT_SUPPORTED, "SDPA FP8 does not support bias");
+
+ // validate basic dimension requirements
+ // d_qk=192 with d_v=128 is only supported starting from cuDNN 9.19
+ bool const d192_v128_supported = (detail::get_backend_version() >= 91900);
+ if (prop_major >= 10) {
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ ((d_qk > 128) || (d_qk % 16 != 0)) && !(d192_v128_supported && d_qk == 192 && d_v == 128),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "hidden_dim d_qk should be less than or equal to 128 and hidden_dim d_qk "
+ "should be multiple of 16 unless d_qk == 192 and d_v == 128 (requires cuDNN 9.19+)");
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ ((d_v > 128) || (d_v % 16 != 0)),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "hidden_dim d_v should be less than or equal to 128 and hidden_dim d_v should be multiple of 16");
+ } else {
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ (d_qk > 256) || (d_qk % 16 != 0) || (d_v > 256) || (d_v % 16 != 0),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "hidden_dim shoud be less than or equal to 256 and hidden_dim should be multiple of 16");
+ }
+
+ // Validate options for causal_mask_bottom_right
+ RETURN_CUDNN_FRONTEND_ERROR_IF(has_causal_mask_bottom_right() && detail::get_backend_version() < 90700,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "For cuDNN version below 9.7.0, bottom right causal masking is not supported.");
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ has_causal_mask_bottom_right() && prop_major < 10,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "sdpa fp8 forward operation is only supported on Blackwell architecture and newer. Please "
+ "consider using a newer architecture.");
+
+ // if output data type is half or bfloat16, and version is below 9.13 or is not blackwell, return NOT_SUPPORTED
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ (output_data_type == DataType_t::HALF || output_data_type == DataType_t::BFLOAT16) &&
+ (detail::get_backend_version() < 91300 || prop_major < 10),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "sdpa fp8 forward with HALF/BFLOAT16 output is only supported on Blackwell architecture "
+ "with cuDNN version 9.13.0 and newer.");
+ } else if (mma_core_mode == DataType_t::HALF) {
+ // FP16 specific validation
+
+ // Bug workarounds for known problematic versions
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ detail::get_backend_version() == 91000 || detail::get_backend_version() == 91001,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "SDPA FP16/BF16 forward is not supported on cuDNN 9.10.0/9.10.1 due to known bugs. "
+ "Please consider upgrading to 9.10.2 or newer.");
+
+ // 9.14.0 sliding window bug: non-causal + s_kv > 1024 + sliding window
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ detail::get_backend_version() == 91400 && s_kv > 1024 && left_bound.has_value() &&
+ !has_causal_like_masking(),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "cuDNN 9.14.0 has a known bug with non-causal + s_kv > 1024 + sliding window attention. "
+ "Please consider upgrading to 9.14.1 or newer.");
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ (attention_score_modifier != nullptr) &&
+ (alibi_mask || has_causal_like_masking() || padding_mask || left_bound.has_value()),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Attention score mod enabled and hence other subgraphs are disabled.");
+
+ // validate basic dimension requirements
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ (d_qk % 8 != 0) || (d_v % 8 != 0), error_code_t::GRAPH_NOT_SUPPORTED, "hidden_dim should be multiple of 8");
+
+ // validate alibi requirements
+ RETURN_CUDNN_FRONTEND_ERROR_IF(alibi_mask && !(right_bound.has_value() && right_bound.value() == 0),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "When alibi mask is used, diagonal_band_right_bound needs to be set to 0.");
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ left_bound.has_value() && left_bound.value() <= 0 && detail::get_backend_version() < 91000,
+ error_code_t::INVALID_VALUE,
+ "Left bound (Sliding window length) should be greater than zero when set.");
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(left_bound.has_value() && (!padding_mask) && s_q > s_kv,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Sliding window attention is only supported with max_s_q <= max_s_kv.");
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ left_bound.has_value() && (s_q * left_bound.value() == s_kv * left_bound.value()) &&
+ (detail::get_backend_version() <= 90900) && (prop_major == 9) && has_causal_mask_bottom_right(),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "On Hopper architecture, this specific combination of s_q, s_kv, and left_bound + right_bound + bottom "
+ "right diagonal alignment is not supported for backend version 9.9 or below");
+
+ if ((detail::get_backend_version() < 91002)) {
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ left_bound.has_value() && (!has_causal_like_masking() || is_dropout || is_bias),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Left and right bounds are only supported with is_dropout=False, is_bias=False. And the diagonal "
+ "alignment must be set.");
+ }
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(right_bound.has_value() && right_bound.value() < 0,
+ error_code_t::INVALID_VALUE,
+ "Right bound needs to be larger than or equal to zero");
+
+ // Validate options for s_q == 1
+ const bool is_decode_only = (s_q == 1);
+ RETURN_CUDNN_FRONTEND_ERROR_IF(is_decode_only && (prop_major == 10) && (d_qk > 128 || d_v > 128) &&
+ (detail::get_backend_version() <= 90900),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "decode only mode, i.e. s_q == 1 not supported for blackwell architecture with "
+ "d_qk or d_v > 128 for backend version 9.9 or below");
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ is_decode_only && (detail::get_backend_version() <= 90900) && (right_bound.has_value()),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "decode only mode, i.e. s_q == 1, not supported with masking (right_bound is set) for backend version 9.9 "
+ "or below");
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(is_decode_only && has_sink_token(),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "decode only mode, i.e. s_q == 1, not supported with sink_token");
+
+ // validate options for paged attention
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ is_paged && (d_qk > 128 || d_v > 128) && detail::get_backend_version() <= 90900,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Paged attention only supported with d_qk and d_v <= 128 for backend version 9.9 or below");
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(is_paged && is_ragged && detail::get_backend_version() < 90700,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Paged caches are not supported in combination with ragged offsets.");
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ is_paged && !((has_seq_len_q && has_seq_len_kv) || (has_cu_seq_len_q && has_cu_seq_len_kv)),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Paged caches can only be used in combination with padding mask and variable sequence lengths "
+ "for both Q and KV (via seq_len_q/seq_len_kv or cu_seq_len_q/cu_seq_len_kv).");
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ !is_paged && max_seq_kv_explicit,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "When not using paged attention, there is no need to explicitly set max kv sequence length.");
+
+ if (max_seq_kv_explicit) {
+ auto max_seq_kv = max_seq_len_kv.value();
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(is_bias && (bias_mask->second->get_dim()[3] != max_seq_kv),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Value set through set_paged_attention_max_seq_len_kv is incompatible with "
+ "the sequence length of the bias");
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(is_rng && rng_tensor->second->get_dim()[3] != max_seq_kv,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Value set through set_paged_attention_max_seq_len_kv is incompatible with "
+ "the sequence length of the RNG_DUMP");
+ }
+
+ // Additional validation for paged attention with packed page tables
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ ((is_paged_k && inputs.at(SDPA_attributes::input_names::Page_table_K)->get_ragged_offset()) ||
+ (is_paged_v && inputs.at(SDPA_attributes::input_names::Page_table_V)->get_ragged_offset())) &&
+ detail::get_backend_version() < 91002,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Paged attention with packed page tables only supported with cudnn version 9.10.2 and above");
+
+ // SM version check for SDPA
+ RETURN_CUDNN_FRONTEND_ERROR_IF(prop_major < 8,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "SDPA FP16/BF16 requires SM80 (Ampere) or newer architecture");
+
+ // version specific validation by architecture
+ if (prop_major == 8) {
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ detail::get_backend_version() <= 90900 && ((d_qk > 128) || (d_v > 128)),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "head_dim should be less than or equal to 128 for backend version 9.9 or below on ampere architecture");
+ }
+ if (prop_major == 9) {
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ detail::get_backend_version() <= 90900 && ((d_qk > 256) || (d_v > 256)),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "head_dim should be less than or equal to 256 for backend version 9.9 or below on hopper architecture");
+ }
+ if (prop_major == 10) {
+ RETURN_CUDNN_FRONTEND_ERROR_IF((detail::get_backend_version() < 90900) && ((d_qk > 128) || (d_v > 128)),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "head_dim should be less than or equal to 128 for backend version 9.8 or "
+ "below on blackwell architecture");
+ }
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 90500 && is_paged,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "For cuDNN version below 9.5.0, paged caches are not supported");
+
+ if (is_ragged) {
+ RETURN_CUDNN_FRONTEND_ERROR_IF((context.get_sm_version() > 0 && context.get_sm_version() < 90 &&
+ detail::get_backend_version() < 91801),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "THD (ragged offset) is only supported in Hopper and above : " +
+ std::to_string(context.get_sm_version()));
+ }
+ } else {
+ RETURN_CUDNN_FRONTEND_ERROR_IF(true, error_code_t::GRAPH_NOT_SUPPORTED, "Unsupported mma core mode");
+ }
+
+ // Check whether the selected implementation supports the requested features.
+ CHECK_CUDNN_FRONTEND_ERROR(verify_sdpa_support_surface_for_implementation(context, implementation));
+
+ return {error_code_t::OK, ""};
+}
+
+// Verify that the underlying implementation supports all the features in these attributes.
+// Unlike `validate_sdpa_support_surface()`, this may be called before validation, so:
+// * don't assume any particular keys already exist in `inputs` or `outputs`
+// * don't assume any tensor dims or strides are already set
+// We return error codes directly instead of using `RETURN_CUDNN_FRONTEND_ERROR_IF`
+// to avoid unneeded logging when this function is being called in a non-error-generating
+// situation (e.g. during auto-select of SDPA implementation).
+inline error_t
+SDPA_attributes::verify_sdpa_support_surface_for_implementation(const detail::Context& context,
+ AttentionImplementation_t impl) const {
+ switch (impl) {
+ case AttentionImplementation_t::AUTO:
+ // This function should not be called with AUTO.
+ return {error_code_t::INVALID_VALUE,
+ "Can't call verify_sdpa_support_surface_for_implementation with impl=AUTO"};
+ case AttentionImplementation_t::COMPOSITE:
+ for (const auto& [key, value] : inputs) {
+ RETURN_CUDNN_FRONTEND_ERROR_IF(key == input_names::Block_mask && value != nullptr,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Composite SDPA node doesn't support Block_mask input");
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ (key == input_names::CU_SEQ_LEN_Q || key == input_names::CU_SEQ_LEN_KV) && value != nullptr,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Composite SDPA node doesn't support CU_SEQ_LEN_Q / CU_SEQ_LEN_KV inputs");
+ }
+ // The ragged offset multiplier is only supported by the unified forward engine.
+ // Reject it here so auto-select routes such graphs to the unified implementation.
+ for (const auto& [key, value] : inputs) {
+ RETURN_CUDNN_FRONTEND_ERROR_IF(value != nullptr && value->has_ragged_offset_multiplier(),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Composite SDPA node doesn't support a ragged offset multiplier");
+ }
+ for (const auto& [key, value] : outputs) {
+ RETURN_CUDNN_FRONTEND_ERROR_IF(value != nullptr && value->has_ragged_offset_multiplier(),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Composite SDPA node doesn't support a ragged offset multiplier");
+ }
+ break;
+ case AttentionImplementation_t::UNIFIED: {
+ auto effective_cudnn_ver = std::min(detail::get_backend_version(), detail::get_compiled_version());
+ RETURN_CUDNN_FRONTEND_ERROR_IF(effective_cudnn_ver < 91301,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Unified SDPA node requires cuDNN 9.13.1");
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(context.get_dynamic_shape_enabled(),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "Unified SDPA node doesn't yet support dynamic shape");
+
+ // TODO: Provide smarter error messages that provide the required cuDNN version for each input.
+ std::unordered_set<SDPA_attributes::input_names> allowed_input_names{
+ input_names::Q, input_names::K, input_names::V, input_names::Attn_scale};
+ std::string allowed_input_msg =
+ "Unified SDPA node doesn't yet support inputs other than Q, K, V, Attn_scale";
+
+ if (effective_cudnn_ver >= 91400) {
+ allowed_input_names.insert({input_names::Block_mask});
+ allowed_input_msg += ", Block_mask";
+ }
+
+ if (effective_cudnn_ver >= 91500) {
+ allowed_input_names.insert({input_names::Page_table_K,
+ input_names::Page_table_V,
+ input_names::SEQ_LEN_Q,
+ input_names::SEQ_LEN_KV});
+ allowed_input_msg += ", Page_table_K, Page_table_V, SEQ_LEN_Q, SEQ_LEN_KV";
+ }
+
+ if (effective_cudnn_ver >= 92100) {
+ // NOTE: For unified engine, we support dropout via Seed and Offset only.
+ // Custom dropout mask (via Dropout_mask, Dropout_scale) is not supported.
+ allowed_input_names.insert(
+ {input_names::Bias, input_names::Seed, input_names::Offset, input_names::SINK_TOKEN});
+ allowed_input_msg += ", Bias, Seed, Offset, SINK_TOKEN";
+ }
+
+ if (effective_cudnn_ver >= 92400) {
+ allowed_input_names.insert({input_names::CU_SEQ_LEN_Q, input_names::CU_SEQ_LEN_KV});
+ allowed_input_msg += ", CU_SEQ_LEN_Q, CU_SEQ_LEN_KV";
+ }
+
+ if (effective_cudnn_ver >= 93000) {
+ allowed_input_names.insert({input_names::Descale_Q,
+ input_names::Descale_K,
+ input_names::Descale_V,
+ input_names::Descale_S,
+ input_names::Scale_S,
+ input_names::Scale_O});
+ allowed_input_msg += ", Descale_Q, Descale_K, Descale_V, Descale_S, Scale_S, Scale_O";
+ }
+
+ for (const auto& [key, value] : inputs) {
+ if (allowed_input_names.find(key) == allowed_input_names.end() && value != nullptr) {
+ return {error_code_t::GRAPH_NOT_SUPPORTED, allowed_input_msg};
+ }
+ }
+
+ std::unordered_set<SDPA_attributes::output_names> allowed_output_names{output_names::O,
+ output_names::Stats};
+ std::string allowed_output_msg = "Unified SDPA node doesn't yet support outputs other than O, Stats";
+
+ if (effective_cudnn_ver >= 92100) {
+ allowed_output_names.insert({output_names::RNG_DUMP, output_names::Max, output_names::Sum_exp});
+ allowed_output_msg += ", RNG_DUMP, Max, Sum_exp";
+ }
+
+ if (effective_cudnn_ver >= 93000) {
+ allowed_output_names.insert({output_names::Amax_S, output_names::Amax_O});
+ allowed_output_msg += ", Amax_S, Amax_O";
+ }
+
+ for (const auto& [key, value] : outputs) {
+ if (allowed_output_names.find(key) == allowed_output_names.end() && value != nullptr) {
+ return {error_code_t::GRAPH_NOT_SUPPORTED, allowed_output_msg};
+ }
+ }
+
+ if (alibi_mask && effective_cudnn_ver < 92100) {
+ return {error_code_t::GRAPH_NOT_SUPPORTED,
+ "Alibi mask for unified SDPA node requires cuDNN 9.21.0 or above"};
+ }
+
+ if (padding_mask && effective_cudnn_ver < 91500) {
+ return {error_code_t::GRAPH_NOT_SUPPORTED,
+ "Padding mask for unified SDPA node requires cuDNN 9.15.0 or above"};
+ }
+
+ if ((left_bound.has_value() || right_bound.has_value()) && effective_cudnn_ver < 92100) {
+ return {error_code_t::GRAPH_NOT_SUPPORTED,
+ "Left bound or right bound for unified SDPA node requires cuDNN 9.21.0 or above"};
+ }
+
+ if (diagonal_alignment != DiagonalAlignment_t::TOP_LEFT && effective_cudnn_ver < 92100) {
+ return {error_code_t::GRAPH_NOT_SUPPORTED,
+ "Diagonal alignment for unified SDPA node requires cuDNN 9.21.0 or above"};
+ }
+
+ if (dropout_probability.has_value() && effective_cudnn_ver < 92200) {
+ return {error_code_t::GRAPH_NOT_SUPPORTED, "Dropout for unified SDPA node requires cuDNN 9.22.0"};
+ }
+
+ if (dropout_probability.has_value() && generate_stats.value_or(false)) {
+ return {error_code_t::GRAPH_NOT_SUPPORTED,
+ "Dropout for unified SDPA node with generated stats is not supported"};
+ }
+
+ // Unified engine in cuDNN < 9.15 can't meaningfully support max sequence length,
+ // while versions >= 9.15 "support" it by ignoring it (unified engine doesn't need it).
+ if (max_seq_len_kv.has_value() && effective_cudnn_ver < 91500) {
+ return {error_code_t::GRAPH_NOT_SUPPORTED,
+ "Max sequence length for unified SDPA node cannot be set in cuDNN < 9.15.0"};
+ }
+
+ if (attention_score_modifier != nullptr && effective_cudnn_ver < 92100) {
+ return {error_code_t::GRAPH_NOT_SUPPORTED,
+ "Attention score modifier for unified SDPA node requires cuDNN 9.21.0 or above"};
+ }
+
+ if (mma_core_mode == DataType_t::FP8_E4M3 || mma_core_mode == DataType_t::FP8_E5M2) {
+ // Per-tensor FP8 and MXFP8 (block-scaled, E8M0 descales) are supported by the
+ // unified node starting from cuDNN 9.30.0.
+ if (effective_cudnn_ver < 93000) {
+ return {error_code_t::GRAPH_NOT_SUPPORTED,
+ "FP8/MXFP8 for the unified SDPA node requires cuDNN 9.30.0 or above"};
+ }
+ }
+ } break;
+ }
+
+ return {error_code_t::OK, ""};
+}
+
+} // namespace cudnn_frontend::graph
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/slice.h b/deps/cudnn-frontend/include/cudnn_frontend/node/slice.h
new file mode 100644
index 0000000..a1aafb1
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/node/slice.h
@@ -0,0 +1,224 @@
+#pragma once
+
+#include "../graph_helpers.h"
+#include "../node_interface.h"
+
+namespace cudnn_frontend::graph {
+
+class SliceNode : public NodeCRTP<SliceNode> {
+ public:
+ Slice_attributes attributes;
+
+ SliceNode(Slice_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP(context), attributes(std::move(attributes_)) {}
+
+ Type
+ getType() override final {
+ return Type::SLICE;
+ }
+
+ error_t
+ infer_properties_node() override final {
+ getLogger() << "[cudnn_frontend] INFO: Inferrencing properties for slice node " << attributes.name
+ << std::endl;
+
+ attributes.fill_from_context(context);
+
+ for (size_t i = 0; i < attributes.slice_strides.size(); ++i) {
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ attributes.slice_strides[i] <= 0,
+ error_code_t::INVALID_VALUE,
+ "Slice slice_strides[" + std::to_string(i) + "] must be strictly positive (got " +
+ std::to_string(attributes.slice_strides[i]) +
+ "). Non-positive strides break output dimension calculation and can cause division by zero.");
+ }
+
+ auto output = attributes.outputs.at(Slice_attributes::output_names::Y);
+ auto output_dim = output->get_dim();
+
+ if (output_dim.empty()) {
+ for (size_t i = 0; i < attributes.slices.size(); ++i) {
+ int64_t start = attributes.slices[i].first;
+ int64_t limit = attributes.slices[i].second;
+ int64_t stride = (!attributes.slice_strides.empty() && i < attributes.slice_strides.size())
+ ? attributes.slice_strides[i]
+ : 1;
+ // Output dimension = ceil((limit - start) / stride)
+ int64_t dim = (limit - start + stride - 1) / stride;
+ output_dim.push_back(dim);
+ }
+ output->set_dim(output_dim);
+ }
+
+ auto const input = attributes.inputs.at(Slice_attributes::input_names::X);
+ auto const input_data_type = input->get_data_type();
+ auto const output_data_type = output->get_data_type();
+ if (output_data_type == DataType_t::NOT_SET) {
+ output->set_data_type(input_data_type);
+ } else {
+ RETURN_CUDNN_FRONTEND_ERROR_IF(output_data_type != input_data_type,
+ error_code_t::INVALID_VALUE,
+ "output and input tensor data types should match for slice operation.");
+ }
+
+ auto const input_stride = input->get_stride();
+ if (output->get_stride().empty()) {
+ // When slice strides > 1, output strides need to be multiplied accordingly
+ std::vector<int64_t> output_stride;
+ for (size_t i = 0; i < input_stride.size(); ++i) {
+ int64_t stride = (!attributes.slice_strides.empty() && i < attributes.slice_strides.size())
+ ? attributes.slice_strides[i]
+ : 1;
+ output_stride.push_back(input_stride[i] * stride);
+ }
+ output->set_stride(output_stride);
+ }
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ create_cudnn_tensors_node(std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors,
+ int64_t& potential_uid,
+ std::unordered_set<int64_t> const& used_uids) const override final {
+ getLogger() << "[cudnn_frontend] INFO: Creating cudnn tensors for SliceNode " << attributes.name << std::endl;
+
+ auto const input = attributes.inputs.at(Slice_attributes::input_names::X);
+ auto const output = attributes.outputs.at(Slice_attributes::output_names::Y);
+
+ if (detail::get_backend_version() >= 92200 && detail::get_compiled_version() >= 92200) {
+ CHECK_CUDNN_FRONTEND_ERROR(detail::create_cudnn_tensor(input, tensors, potential_uid, used_uids));
+ CHECK_CUDNN_FRONTEND_ERROR(detail::create_cudnn_tensor(output, tensors, potential_uid, used_uids));
+ return {error_code_t::OK, ""};
+ }
+
+ if (input->has_uid() == false) {
+ detail::assign_uid(input.get(), potential_uid, used_uids);
+ }
+ output->set_is_virtual(false);
+ CHECK_CUDNN_FRONTEND_ERROR(detail::create_cudnn_tensor(output, tensors, potential_uid, used_uids));
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ create_cudnn_operations(
+ std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations,
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations,
+ managed_backend_descriptor_t& raw_operations,
+ std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final {
+ getLogger() << "[cudnn_frontend] INFO: " << "Building SliceNode operations " << attributes.name << std::endl;
+
+#if (CUDNN_VERSION >= 92200)
+ // cuDNN >= 9.22.0: Use native backend slice operation
+ auto cudnn_ver_error = error_t{error_code_t::GRAPH_NOT_SUPPORTED, "Slice backend operation requires 9.22.0"};
+ NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(92200, cudnn_ver_error);
+
+ if (detail::get_backend_version() >= 92200) {
+ CUDNN_FRONTEND_UNUSED(operations);
+
+ auto slice_operation = make_shared_backend_pointer(CUDNN_BACKEND_OPERATION_SLICE_DESCRIPTOR);
+
+ // Set input tensor
+ auto X = attributes.inputs.at(Slice_attributes::input_names::X);
+ auto backend_x = tensors[X->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(slice_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_SLICE_XDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_x));
+
+ // Set output tensor
+ auto Y = attributes.outputs.at(Slice_attributes::output_names::Y);
+ auto backend_y = tensors[Y->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(slice_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_SLICE_YDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_y));
+
+ // Extract start and limit indices from slices
+ std::vector<int64_t> start_indices;
+ std::vector<int64_t> limit_indices;
+
+ for (const auto& slice : attributes.slices) {
+ start_indices.push_back(slice.first);
+ limit_indices.push_back(slice.second);
+ }
+
+ // Per-dimension strides: use user slice_strides[i] when set, else 1 (preserves partial configuration)
+ std::vector<int64_t> strides(attributes.slices.size());
+ for (size_t i = 0; i < strides.size(); ++i) {
+ strides[i] = (i < attributes.slice_strides.size()) ? attributes.slice_strides[i] : 1;
+ }
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(slice_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_SLICE_START_INDICES,
+ CUDNN_TYPE_INT64,
+ start_indices.size(),
+ start_indices.data()));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(slice_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_SLICE_LIMIT_INDICES,
+ CUDNN_TYPE_INT64,
+ limit_indices.size(),
+ limit_indices.data()));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(slice_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_SLICE_STRIDES,
+ CUDNN_TYPE_INT64,
+ strides.size(),
+ strides.data()));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(slice_operation->get_backend_descriptor()));
+
+ raw_operations.push_back(slice_operation);
+
+ auto const& non_virtual_uids = attributes.get_non_virtual_uids();
+ uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end());
+ return {error_code_t::OK, ""};
+ }
+#endif
+ CUDNN_FRONTEND_UNUSED(operations);
+ CUDNN_FRONTEND_UNUSED(raw_operations);
+ CUDNN_FRONTEND_UNUSED(tensors);
+
+ getLogger() << "[cudnn_frontend] INFO: " << "Using pointer arithmetic fallback for slice on cuDNN < 9.22.0"
+ << std::endl;
+
+ auto const output = attributes.outputs.at(Slice_attributes::output_names::Y);
+ if (output && output->get_is_virtual() == false) {
+ uids_involved_in_operations.insert(output->get_uid());
+ if (auto ragged_offset = output->get_ragged_offset()) {
+ uids_involved_in_operations.insert(ragged_offset->get_uid());
+ }
+ }
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ collect_variant_pack_replacements_node(
+ std::unordered_map<Tensor_attributes::uid_t, std::pair<Tensor_attributes::uid_t, int64_t>>&
+ variant_pack_replacements) const override final {
+ if (detail::get_backend_version() >= 92200 && detail::get_compiled_version() >= 92200) {
+ CUDNN_FRONTEND_UNUSED(variant_pack_replacements);
+ return {error_code_t::OK, ""};
+ }
+ auto const input = attributes.inputs.at(Slice_attributes::input_names::X);
+ auto const output = attributes.outputs.at(Slice_attributes::output_names::Y);
+
+ variant_pack_replacements[input->get_uid()] = {output->get_uid(), attributes.get_offset()};
+
+ return {error_code_t::OK, ""};
+ };
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ j.update(R"( {"tag": "SLICE"})"_json);
+ }
+#endif
+};
+
+} // namespace cudnn_frontend::graph \ No newline at end of file
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/softmax.h b/deps/cudnn-frontend/include/cudnn_frontend/node/softmax.h
new file mode 100644
index 0000000..1aa5689
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/node/softmax.h
@@ -0,0 +1,366 @@
+#pragma once
+
+#include "../../cudnn_frontend_Heuristics.h"
+#include "../../cudnn_frontend_Logging.h"
+
+#include "../graph_helpers.h"
+#include "../node_interface.h"
+
+#include "pointwise.h"
+#include "reduction.h"
+
+namespace cudnn_frontend::graph {
+
+// Base class for nodes that represent a softmax operation.
+// This is either implemented as a composite of pointwise/reduction operations (CompositeSoftmaxNode)
+// or as a single unified backend operation (UnifiedSoftmaxNode).
+template <typename DerivedT>
+class SoftmaxNodeBase : public NodeCRTP<DerivedT> {
+ protected:
+ using input_names = Softmax_attributes::input_names;
+ using output_names = Softmax_attributes::output_names;
+
+ public:
+ Softmax_attributes attributes;
+
+ SoftmaxNodeBase(Softmax_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP<DerivedT>(context), attributes(std::move(attributes_)) {}
+
+ error_t
+ infer_properties_node() override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for Softmax node " << attributes.name);
+
+ attributes.fill_from_context(this->context);
+
+ auto input_tensor = attributes.inputs.at(Softmax_attributes::input_names::P);
+ auto output_tensor = attributes.outputs.at(Softmax_attributes::output_names::S);
+
+ // Only infer dims and strides if user did not set them
+ if (output_tensor->get_dim().empty()) {
+ output_tensor->set_dim(input_tensor->get_dim());
+ }
+
+ if (output_tensor->get_stride().empty()) {
+ output_tensor->set_stride(input_tensor->get_stride());
+ }
+
+ return {error_code_t::OK, ""};
+ }
+
+ protected:
+ bool
+ has_sink() const {
+ auto sink_it = attributes.inputs.find(Softmax_attributes::input_names::SINK);
+ return ((sink_it) != attributes.inputs.end() && sink_it->second != nullptr);
+ }
+
+ bool
+ has_stats() const {
+ auto stats_it = attributes.outputs.find(Softmax_attributes::output_names::Stats);
+ return ((stats_it) != attributes.outputs.end() && stats_it->second != nullptr);
+ }
+
+ bool
+ has_max() const {
+ auto max_it = attributes.outputs.find(Softmax_attributes::output_names::Max);
+ return ((max_it) != attributes.outputs.end() && max_it->second != nullptr);
+ }
+
+ bool
+ has_sum_exp() const {
+ auto sum_exp_it = attributes.outputs.find(Softmax_attributes::output_names::Sum_exp);
+ return ((sum_exp_it) != attributes.outputs.end() && sum_exp_it->second != nullptr);
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ void
+ serialize(json& j) const override final {
+ j = attributes;
+ }
+#endif
+};
+
+// Fallback implementation of softmax node that represents the operation as a series of pointwise and reduction
+// operations. This is used for cuDNN versions before v9.21.0.
+// Only certain combinations of outputs are allowed.
+class CompositeSoftmaxNode : public SoftmaxNodeBase<CompositeSoftmaxNode> {
+ public:
+ CompositeSoftmaxNode(Softmax_attributes&& attributes_, detail::Context const& context)
+ : SoftmaxNodeBase(std::move(attributes_), context) {}
+
+ Type
+ getType() override final {
+ return Type::COMPOSITE;
+ }
+
+ error_t
+ pre_validate_node() const override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating CompositeSoftmaxNode " << attributes.name);
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ !((!has_stats() && !has_max() && !has_sum_exp()) || //
+ (has_stats() && !has_max() && !has_sum_exp()) || //
+ (!has_stats() && has_max() && has_sum_exp())),
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "CompositeSoftmaxNode can only output certain combinations of stats, max and sum_exp: "
+ "stats only, max and sum_exp only, or none of the above.");
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ expand_node() override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for CompositeSoftmaxNode " << attributes.name);
+
+ attributes.fill_from_context(context);
+
+ // Fill properties of virtual tensors
+ auto const p_dim = attributes.inputs[Softmax_attributes::input_names::P]->get_dim();
+ auto b = p_dim[0];
+ auto h = p_dim[1];
+ auto s_q = p_dim[2];
+
+ std::shared_ptr<Tensor_attributes> max_output;
+ if (has_max()) {
+ max_output = attributes.outputs[Softmax_attributes::output_names::Max];
+ } else {
+ max_output = std::make_shared<Tensor_attributes>();
+ max_output->set_is_virtual(true);
+ }
+ //////////////// TODO //////////////////////////
+ // Check Stride (Before setting dimension?)
+ if (max_output->get_dim().empty()) {
+ max_output->set_dim({b, h, s_q, 1});
+ }
+ if (max_output->get_stride().empty()) {
+ max_output->set_stride({h * s_q, s_q, 1, 1});
+ }
+
+ auto max_attributes = Reduction_attributes().set_name("Max").set_mode(ReductionMode_t::MAX);
+ // If sink tensor is present, we also need to take a pointwise max with sink
+ if (has_sink()) {
+ auto s_max = reduction(attributes.inputs[Softmax_attributes::input_names::P], max_attributes);
+ s_max->set_name("s_max");
+
+ auto sink_tensor = attributes.inputs[Softmax_attributes::input_names::SINK];
+ auto sink_attributes = Pointwise_attributes().set_name("max_sink").set_mode(PointwiseMode_t::MAX);
+ pointwise(s_max, sink_tensor, sink_attributes, max_output);
+ } else {
+ // Special non-functional-style call. Needed because output already created and provided to user.
+ reduction(attributes.inputs[Softmax_attributes::input_names::P], max_attributes, max_output);
+ }
+
+ auto sub_attributes = Pointwise_attributes().set_name("sub").set_mode(PointwiseMode_t::SUB);
+ auto const& sub_output =
+ pointwise(attributes.inputs[Softmax_attributes::input_names::P], max_output, sub_attributes);
+ sub_output->set_name("sub_M");
+
+ auto exp_attributes = Pointwise_attributes().set_name("exp").set_mode(PointwiseMode_t::EXP);
+ auto const& exp_output = pointwise(sub_output, exp_attributes);
+ exp_output->set_name("exp_sub_M");
+
+ std::shared_ptr<Tensor_attributes> sum_output;
+ if (has_sum_exp()) {
+ sum_output = attributes.outputs[Softmax_attributes::output_names::Sum_exp];
+ } else {
+ sum_output = std::make_shared<Tensor_attributes>();
+ sum_output->set_is_virtual(true);
+ }
+ sum_output->set_name("SumExp");
+ if (sum_output->get_dim().empty()) {
+ sum_output->set_dim({b, h, s_q, 1});
+ }
+ if (sum_output->get_stride().empty()) {
+ sum_output->set_stride({h * s_q, s_q, 1, 1});
+ }
+ auto sum_attributes = Reduction_attributes().set_name("sum").set_mode(ReductionMode_t::ADD);
+ // If sink tensor is present, also subtract it and take its exp
+ if (has_sink()) {
+ auto sink_tensor = attributes.inputs[Softmax_attributes::input_names::SINK];
+ auto sub_sink = pointwise(sink_tensor, max_output, sub_attributes);
+ sub_sink->set_name("sub_sink");
+
+ auto exp_sink = pointwise(sub_sink, exp_attributes);
+ exp_sink->set_name("exp_sink");
+
+ auto temp_sum = reduction(exp_output, sum_attributes);
+ temp_sum->set_name("SumExp_elements").set_dim({b, h, s_q, 1}).set_stride({h * s_q, s_q, 1, 1});
+
+ auto add_attributes = Pointwise_attributes().set_name("add_sink").set_mode(PointwiseMode_t::ADD);
+ pointwise(temp_sum, exp_sink, add_attributes, sum_output);
+ } else {
+ reduction(exp_output, sum_attributes, sum_output);
+ }
+
+ // Set to virtual when:
+ // - softmax stats in not requested
+ // - max and sum_exp are not requested
+ if (!has_stats() && !has_max() && !has_sum_exp()) {
+ auto softmax_stats = std::make_shared<Tensor_attributes>();
+ softmax_stats->set_is_virtual(true);
+ attributes.outputs[Softmax_attributes::output_names::Stats] = softmax_stats;
+ }
+
+ if (has_stats()) {
+ auto log_attributes = Pointwise_attributes().set_name("log").set_mode(PointwiseMode_t::LOG);
+ auto const& log_output = pointwise(sum_output, log_attributes);
+ log_output->set_dim({b, h, s_q, 1}).set_stride({h * s_q, s_q, 1, 1});
+
+ auto add_attributes = Pointwise_attributes().set_name("add").set_mode(PointwiseMode_t::ADD);
+ // Special non-functional-style call. Needed because output already created and provided to user.
+ pointwise(
+ max_output, log_output, add_attributes, attributes.outputs[Softmax_attributes::output_names::Stats]);
+ }
+
+ auto div_attributes = Pointwise_attributes().set_name("div").set_mode(PointwiseMode_t::DIV);
+ // Special non-functional-style call. Needed because output already created and provided to user.
+ pointwise(exp_output, sum_output, div_attributes, attributes.outputs[Softmax_attributes::output_names::S]);
+
+ return {error_code_t::OK, ""};
+ }
+};
+
+// Newer implementation of softmax node that represents the operation as a single backend operation.
+// This is used for cuDNN versions v9.21.0 and later.
+class UnifiedSoftmaxNode : public SoftmaxNodeBase<UnifiedSoftmaxNode> {
+ public:
+ UnifiedSoftmaxNode(Softmax_attributes&& attributes_, detail::Context const& context)
+ : SoftmaxNodeBase(std::move(attributes_), context) {}
+
+ Type
+ getType() override final {
+ return Type::SOFTMAX;
+ }
+
+ error_t
+ pre_validate_node() const override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating UnifiedSoftmaxNode " << attributes.name);
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ create_cudnn_operations(
+ std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations,
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations,
+ managed_backend_descriptor_t& raw_operations,
+ std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final {
+ getLogger() << "[cudnn_frontend] INFO: " << "Building UnifiedSoftmaxNode operations " << attributes.name
+ << std::endl;
+ auto cudnn_ver_error = error_t{error_code_t::GRAPH_NOT_SUPPORTED, "UnifiedSoftmaxNode requires cuDNN v9.21.0"};
+
+#if (CUDNN_VERSION >= 92100)
+ NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(92100, cudnn_ver_error);
+ CUDNN_FRONTEND_UNUSED(operations);
+ auto softmax_operation =
+ make_shared_backend_pointer((cudnnBackendDescriptorType_t)CUDNN_BACKEND_OPERATION_SOFTMAX_DESCRIPTOR);
+
+ // Set input tensor P
+ auto P = attributes.inputs.find(Softmax_attributes::input_names::P)->second;
+ auto backend_p = tensors[P->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(softmax_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_SOFTMAX_XDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_p));
+
+ // Set output tensor S
+ auto S = attributes.outputs.find(Softmax_attributes::output_names::S)->second;
+ auto backend_s = tensors[S->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(softmax_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_SOFTMAX_YDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_s));
+
+ // Set optional SINK tensor if present
+ if (has_sink()) {
+ auto SINK = attributes.inputs.find(Softmax_attributes::input_names::SINK)->second;
+ auto backend_sink = tensors[SINK->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(softmax_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_SOFTMAX_SINK_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_sink));
+ }
+
+ // Set optional Stats output tensor if present
+ if (has_stats()) {
+ auto Stats = attributes.outputs.find(Softmax_attributes::output_names::Stats)->second;
+ auto backend_stats = tensors[Stats->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(softmax_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_SOFTMAX_STATS_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_stats));
+ }
+
+ // Set optional Max output tensor if present
+ if (has_max()) {
+ auto Max = attributes.outputs.find(Softmax_attributes::output_names::Max)->second;
+ auto backend_max = tensors[Max->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(softmax_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_SOFTMAX_MAX_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_max));
+ }
+
+ // Set optional Sum_exp output tensor if present
+ if (has_sum_exp()) {
+ auto Sum_exp = attributes.outputs.find(Softmax_attributes::output_names::Sum_exp)->second;
+ auto backend_sum_exp = tensors[Sum_exp->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(softmax_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_SOFTMAX_SUM_EXP_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_sum_exp));
+ }
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(softmax_operation->get_backend_descriptor()));
+
+ raw_operations.push_back(softmax_operation);
+
+ auto const& non_virtual_uids = attributes.get_non_virtual_uids();
+ uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end());
+ return {error_code_t::OK, ""};
+#else
+ CUDNN_FRONTEND_UNUSED(uids_involved_in_operations);
+ CUDNN_FRONTEND_UNUSED(operations);
+ CUDNN_FRONTEND_UNUSED(raw_operations);
+ CUDNN_FRONTEND_UNUSED(tensors);
+ return cudnn_ver_error;
+#endif // CUDNN_VERSION >= 92100
+ }
+};
+
+// Factory function implementation for INode::softmax
+// This function will create the appropriate node based on the cuDNN version.
+inline void
+INode::softmax(std::shared_ptr<Tensor_attributes> p,
+ Softmax_attributes attributes,
+ std::shared_ptr<Tensor_attributes> s,
+ std::shared_ptr<Tensor_attributes> stats,
+ std::shared_ptr<Tensor_attributes> max,
+ std::shared_ptr<Tensor_attributes> sum_exp) {
+ attributes.inputs[Softmax_attributes::input_names::P] = p;
+ attributes.outputs[Softmax_attributes::output_names::S] = s;
+ if (stats) {
+ attributes.outputs[Softmax_attributes::output_names::Stats] = stats;
+ }
+ if (max) {
+ attributes.outputs[Softmax_attributes::output_names::Max] = max;
+ }
+ if (sum_exp) {
+ attributes.outputs[Softmax_attributes::output_names::Sum_exp] = sum_exp;
+ }
+
+ // Newer versions of cuDNN can represent the softmax as a single operation.
+ if (std::min(detail::get_compiled_version(), detail::get_backend_version()) >= 92100) {
+ sub_nodes.emplace_back(std::make_unique<UnifiedSoftmaxNode>(std::move(attributes), context));
+ } else {
+ sub_nodes.emplace_back(std::make_unique<CompositeSoftmaxNode>(std::move(attributes), context));
+ }
+}
+
+} // namespace cudnn_frontend::graph
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/transpose.h b/deps/cudnn-frontend/include/cudnn_frontend/node/transpose.h
new file mode 100644
index 0000000..7f764be
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/node/transpose.h
@@ -0,0 +1,164 @@
+#pragma once
+
+#include "../../cudnn_frontend_Logging.h"
+
+#include "../graph_helpers.h"
+#include "../node_interface.h"
+
+namespace cudnn_frontend::graph {
+
+class TransposeNode : public NodeCRTP<TransposeNode> {
+ public:
+ Transpose_attributes attributes;
+
+ TransposeNode(Transpose_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP(context), attributes(std::move(attributes_)) {}
+
+ Type
+ getType() override final {
+ return Type::TRANSPOSE;
+ }
+
+ error_t
+ infer_properties_node() override final {
+ getLogger() << "[cudnn_frontend] INFO: Inferrencing properties for transpose node " << attributes.name
+ << std::endl;
+
+ attributes.fill_from_context(context);
+
+ auto const& X_tensor = attributes.inputs[Transpose_attributes::input_names::X];
+ auto Y_tensor = attributes.outputs[Transpose_attributes::output_names::Y];
+
+ // Get input properties
+ auto const& input_dim = X_tensor->get_dim();
+ auto const& input_stride = X_tensor->get_stride();
+ auto const& input_data_type = X_tensor->get_data_type();
+ auto const& permutation = attributes.get_permutation();
+
+ // Validate permutation
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ permutation.empty(), error_code_t::ATTRIBUTE_NOT_SET, "Permutation must be set for transpose operation.");
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(permutation.size() != input_dim.size(),
+ error_code_t::INVALID_VALUE,
+ "Permutation size must match input tensor dimensionality.");
+
+ // Check that permutation is a valid permutation (contains each index 0 to n-1 exactly once)
+ std::vector<bool> seen(permutation.size(), false);
+ for (auto idx : permutation) {
+ RETURN_CUDNN_FRONTEND_ERROR_IF(idx < 0 || idx >= static_cast<int64_t>(permutation.size()),
+ error_code_t::INVALID_VALUE,
+ "Permutation indices must be in range [0, n-1].");
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ seen[idx], error_code_t::INVALID_VALUE, "Permutation indices must be unique.");
+ seen[idx] = true;
+ }
+
+ // Infer output dimensions by permuting input dimensions
+ std::vector<int64_t> output_dim(input_dim.size());
+ for (size_t i = 0; i < permutation.size(); ++i) {
+ output_dim[i] = input_dim[permutation[i]];
+ }
+
+ // Infer output strides by permuting input strides
+ std::vector<int64_t> output_stride(input_stride.size());
+ for (size_t i = 0; i < permutation.size(); ++i) {
+ output_stride[i] = input_stride[permutation[i]];
+ }
+
+ // Set output tensor properties
+ if (Y_tensor->get_dim().empty()) {
+ Y_tensor->set_dim(output_dim);
+ }
+ if (Y_tensor->get_stride().empty()) {
+ Y_tensor->set_stride(output_stride);
+ }
+ if (Y_tensor->get_data_type() == DataType_t::NOT_SET) {
+ Y_tensor->set_data_type(input_data_type);
+ }
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(Y_tensor->get_data_type() != input_data_type,
+ error_code_t::INVALID_VALUE,
+ "Output and input tensor data types must match for transpose operation.");
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ create_cudnn_operations(
+ std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations,
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations,
+ managed_backend_descriptor_t& raw_operations,
+ std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final {
+ getLogger() << "[cudnn_frontend] INFO: " << "Building TransposeNode operations " << attributes.name
+ << std::endl;
+
+#if (CUDNN_VERSION >= 92200)
+ // cuDNN >= 9.22.0: Use native backend transpose operation
+ auto cudnn_ver_error = error_t{error_code_t::GRAPH_NOT_SUPPORTED, "Transpose requires cuDNN v9.22.0"};
+ NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(92200, cudnn_ver_error);
+ CUDNN_FRONTEND_UNUSED(operations);
+
+ auto transpose_operation = make_shared_backend_pointer(CUDNN_BACKEND_OPERATION_TRANSPOSE_DESCRIPTOR);
+
+ // Set input tensor
+ auto X = attributes.inputs.at(Transpose_attributes::input_names::X);
+ auto backend_x = tensors[X->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(transpose_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_TRANSPOSE_XDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_x));
+
+ // Set output tensor
+ auto Y = attributes.outputs.at(Transpose_attributes::output_names::Y);
+ auto backend_y = tensors[Y->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(transpose_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_TRANSPOSE_YDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_y));
+
+ // Set permutation
+ auto permutation = attributes.get_permutation();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(transpose_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_TRANSPOSE_PERMUTATION,
+ CUDNN_TYPE_INT64,
+ permutation.size(),
+ permutation.data()));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(transpose_operation->get_backend_descriptor()));
+
+ raw_operations.push_back(transpose_operation);
+
+ auto const& non_virtual_uids = attributes.get_non_virtual_uids();
+ uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end());
+#else
+ CUDNN_FRONTEND_UNUSED(operations);
+ CUDNN_FRONTEND_UNUSED(raw_operations);
+ CUDNN_FRONTEND_UNUSED(tensors);
+ CUDNN_FRONTEND_UNUSED(uids_involved_in_operations);
+ return {error_code_t::GRAPH_NOT_SUPPORTED, "Transpose operation requires cuDNN version >= 9.22.0"};
+#endif
+ return {error_code_t::OK, ""};
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ j.update(R"( {"tag": "TRANSPOSE"})"_json);
+ }
+#endif
+};
+
+inline std::shared_ptr<Tensor_attributes>
+INode::transpose(std::shared_ptr<Tensor_attributes> input, Transpose_attributes attributes) {
+ attributes.inputs[Transpose_attributes::input_names::X] = input;
+ auto Y = attributes.outputs[Transpose_attributes::output_names::Y] = output_tensor(attributes.name + "::Y");
+
+ sub_nodes.emplace_back(std::make_unique<TransposeNode>(std::move(attributes), context));
+ return Y;
+}
+
+} // namespace cudnn_frontend::graph