diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-07-02 19:44:08 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-07-02 19:44:08 +0200 |
| commit | 6a10d3577232f1fd646a20f22a4a1ab4ec29f28a (patch) | |
| tree | e4a077b064876ddb39144935fa256a4ab0692488 /deps/cudnn-frontend/include/cudnn_frontend/node | |
| parent | 98a713e86c23bc6098bf7a5558507a9f27a7a1c1 (diff) | |
| download | ffwd-6a10d3577232f1fd646a20f22a4a1ab4ec29f28a.tar.gz ffwd-6a10d3577232f1fd646a20f22a4a1ab4ec29f28a.zip | |
cuda: add cuDNN fused attention path with runtime fallback
Diffstat (limited to 'deps/cudnn-frontend/include/cudnn_frontend/node')
36 files changed, 12351 insertions, 0 deletions
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/adaptive_layernorm.h b/deps/cudnn-frontend/include/cudnn_frontend/node/adaptive_layernorm.h new file mode 100644 index 0000000..86eaa6d --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/adaptive_layernorm.h @@ -0,0 +1,454 @@ +#pragma once + +#include "../../cudnn_frontend_Heuristics.h" +#include "../../cudnn_frontend_Logging.h" + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend { + +namespace graph { +class AdaLayerNormNode : public NodeCRTP<AdaLayerNormNode> { + public: + AdaLayernorm_attributes attributes; + + AdaLayerNormNode(AdaLayernorm_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::ADALAYERNORM; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for adalayernorm node " << attributes.name); + + attributes.fill_from_context(context); + + auto X = attributes.inputs[AdaLayernorm_attributes::input_names::X]; + auto Y = attributes.outputs[AdaLayernorm_attributes::output_names::Y]; + + // Only infer dims and strides if user did not set them + if (Y->get_dim().empty()) { + Y->set_dim(X->get_dim()); + } + if (Y->get_stride().empty()) { + Y->set_stride(X->get_stride()); + } + + // scale_bias dim is 1,c,h,w + auto scale_bias_dim = X->get_dim(); + scale_bias_dim[0] = 1; + + auto scale = attributes.inputs[AdaLayernorm_attributes::input_names::SCALE]; + // Only infer dims and strides if user did not set them + if (scale->get_dim().empty()) { + scale->set_dim(scale_bias_dim); + } + if (scale->get_stride().empty()) { + auto const& scale_dim = scale->get_dim(); + std::vector<int64_t> stride_order; + CHECK_CUDNN_FRONTEND_ERROR( + detail::generate_stride_order_preserving_format(X->get_stride(), scale_dim.size(), stride_order)); + scale->set_stride(detail::generate_stride(scale_dim, stride_order)); + } + + auto bias = attributes.inputs[AdaLayernorm_attributes::input_names::BIAS]; + // Only infer dims and strides if user did not set them + if (bias->get_dim().empty()) { + bias->set_dim(scale_bias_dim); + } + if (bias->get_stride().empty()) { + auto const& bias_dim = bias->get_dim(); + std::vector<int64_t> stride_order; + CHECK_CUDNN_FRONTEND_ERROR( + detail::generate_stride_order_preserving_format(X->get_stride(), bias_dim.size(), stride_order)); + bias->set_stride(detail::generate_stride(bias_dim, stride_order)); + } + + if (attributes.forward_phase == NormFwdPhase_t::TRAINING) { + // stats dim is x where scale == 1 else 1 + auto stats_dim = X->get_dim(); + for (size_t i = 1; i < stats_dim.size(); i++) { + if (scale->get_dim()[i] != 1) { + stats_dim[i] = 1; + } + } + + auto mean = attributes.outputs[AdaLayernorm_attributes::output_names::MEAN]; + // Only infer dims and strides if user did not set them + if (mean->get_dim().empty()) { + mean->set_dim(stats_dim); + } + if (mean->get_stride().empty()) { + auto const& mean_dim = mean->get_dim(); + std::vector<int64_t> stride_order; + CHECK_CUDNN_FRONTEND_ERROR( + detail::generate_stride_order_preserving_format(X->get_stride(), mean_dim.size(), stride_order)); + mean->set_stride(detail::generate_stride(mean_dim, stride_order)); + } + + auto inv_var = attributes.outputs[AdaLayernorm_attributes::output_names::INV_VARIANCE]; + // Only infer dims and strides if user did not set them + if (inv_var->get_dim().empty()) { + inv_var->set_dim(stats_dim); + } + if (inv_var->get_stride().empty()) { + auto const& inv_var_dim = inv_var->get_dim(); + std::vector<int64_t> stride_order; + CHECK_CUDNN_FRONTEND_ERROR( + detail::generate_stride_order_preserving_format(X->get_stride(), inv_var_dim.size(), stride_order)); + inv_var->set_stride(detail::generate_stride(inv_var_dim, stride_order)); + } + } + + // Set scalar tensors + std::vector<int64_t> ones(X->get_dim().size(), 1); + auto infer_scalar_tensors = [&ones](std::shared_ptr<Tensor_attributes>& T) { + // Only infer dims and strides if user did not set them + if (T->get_dim().empty()) { + T->set_dim(ones); + } + if (T->get_stride().empty()) { + T->set_stride(ones); + } + }; + infer_scalar_tensors(attributes.inputs[AdaLayernorm_attributes::input_names::EPSILON]); + + return {error_code_t::OK, ""}; + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: " << "Validating AdaLayerNormNode " << attributes.name); + // Norm forward phase should be set + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.forward_phase == NormFwdPhase_t::NOT_SET, + error_code_t::ATTRIBUTE_NOT_SET, + "Forward phase not set of adalayernorm node."); + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations, + std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final { + getLogger() << "[cudnn_frontend] INFO: " << "Building AdaLayernorm operations " << attributes.name << std::endl; + + auto cudnn_ver_error = error_t{error_code_t::GRAPH_NOT_SUPPORTED, "AdaLN fwd requires cuDNN v9.9.0"}; +#if (CUDNN_VERSION >= 90900) + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(90900, cudnn_ver_error); + CUDNN_FRONTEND_UNUSED(operations); + auto adalayernorm_operation = + make_shared_backend_pointer((cudnnBackendDescriptorType_t)CUDNN_BACKEND_OPERATION_NORM_FORWARD_DESCRIPTOR); + + cudnnBackendNormMode_t cudnn_norm_mode; + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormMode_t::ADA_LAYER_NORM, cudnn_norm_mode)); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_NORM_FWD_MODE, + CUDNN_TYPE_NORM_MODE, + 1, + &cudnn_norm_mode)); + + cudnnBackendNormFwdPhase_t cudnn_norm_fwd_phase; + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.forward_phase, cudnn_norm_fwd_phase)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_NORM_FWD_PHASE, + CUDNN_TYPE_NORM_FWD_PHASE, + 1, + &cudnn_norm_fwd_phase)); + + auto X = attributes.inputs.find(AdaLayernorm_attributes::input_names::X)->second; + auto backend_x = tensors[X->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_NORM_FWD_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_x)); + + auto Scale = attributes.inputs.find(AdaLayernorm_attributes::input_names::SCALE)->second; + auto backend_scale = tensors[Scale->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_NORM_FWD_SCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_scale)); + + auto Bias_iter = attributes.inputs.find(AdaLayernorm_attributes::input_names::BIAS); + if (Bias_iter != attributes.inputs.end() && Bias_iter->second->get_is_virtual() == false) { + auto backend_bias = tensors[Bias_iter->second->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_NORM_FWD_BIAS_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_bias)); + } + + auto Epsilon = attributes.inputs.find(AdaLayernorm_attributes::input_names::EPSILON)->second; + auto backend_epsilon = tensors[Epsilon->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_NORM_FWD_EPSILON_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_epsilon)); + + auto Y = attributes.outputs.find(AdaLayernorm_attributes::output_names::Y)->second; + auto backend_y = tensors[Y->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_NORM_FWD_YDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_y)); + + if (attributes.forward_phase == NormFwdPhase_t::TRAINING) { + auto Mean = attributes.outputs.find(AdaLayernorm_attributes::output_names::MEAN)->second; + auto backend_mean = tensors[Mean->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_NORM_FWD_MEAN_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_mean)); + + auto Inv_variance = attributes.outputs.find(AdaLayernorm_attributes::output_names::INV_VARIANCE)->second; + auto backend_inv_variance = tensors[Inv_variance->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_NORM_FWD_INV_VARIANCE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_inv_variance)); + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(adalayernorm_operation->get_backend_descriptor())); + + raw_operations.push_back(adalayernorm_operation); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; +#else + CUDNN_FRONTEND_UNUSED(uids_involved_in_operations); + CUDNN_FRONTEND_UNUSED(operations); + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FRONTEND_UNUSED(tensors); + return cudnn_ver_error; +#endif + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "ADA_LAYER_NORM"})"_json); + } +#endif +}; + +/*******/ + +class DAdaLayerNormNode : public NodeCRTP<DAdaLayerNormNode> { + public: + AdaLayernorm_backward_attributes attributes; + + DAdaLayerNormNode(AdaLayernorm_backward_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::DADALAYERNORM; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for DAdaLayerNorm node " << attributes.name); + + attributes.fill_from_context(context); + + // TODO: Only inferencing from X works today. + auto X = attributes.inputs[AdaLayernorm_backward_attributes::input_names::X]; + auto const x_tensor_dim = X->get_dim(); + + auto DY = attributes.inputs[AdaLayernorm_backward_attributes::input_names::DY]; + auto dy_tensor_dim = DY->get_dim(); + + // Only infer dims and strides if user did not set them + if (dy_tensor_dim.empty()) { + dy_tensor_dim.resize(x_tensor_dim.size()); + DY->set_dim(x_tensor_dim); + } + if (DY->get_stride().empty()) { + auto const& DY_dim = DY->get_dim(); + // Default to NCHW + auto const& stride_order = detail::generate_row_major_stride_order(DY_dim.size()); + DY->set_stride(detail::generate_stride(DY_dim, stride_order)); + } + + auto DX = attributes.outputs[AdaLayernorm_backward_attributes::output_names::DX]; + auto dx_tensor_dim = DX->get_dim(); + // Only infer dims and strides if user did not set them + if (dx_tensor_dim.empty()) { + dx_tensor_dim.resize(x_tensor_dim.size()); + DX->set_dim(x_tensor_dim); + } + if (DX->get_stride().empty()) { + auto const& DX_dim = DX->get_dim(); + // Default to NCHW + auto const& stride_order = detail::generate_row_major_stride_order(DX_dim.size()); + DX->set_stride(detail::generate_stride(DX_dim, stride_order)); + } + + auto SCALE = attributes.inputs[AdaLayernorm_backward_attributes::input_names::SCALE]; + auto scale_bias_dim = SCALE->get_dim(); + + // Set channel length tensors + auto infer_scale_bias_tensors = [&scale_bias_dim](std::shared_ptr<Tensor_attributes>& T) { + auto tensor_dim = T->get_dim(); + // Only infer dims and strides if user did not set them + if (tensor_dim.empty()) { + T->set_dim(scale_bias_dim); + } + if (T->get_stride().empty()) { + auto const& T_dim = T->get_dim(); + // Default to NCHW + auto const& stride_order = detail::generate_row_major_stride_order(T_dim.size()); + T->set_stride(detail::generate_stride(T_dim, stride_order)); + } + }; + + infer_scale_bias_tensors(attributes.outputs[AdaLayernorm_backward_attributes::output_names::DSCALE]); + auto DBIAS = attributes.outputs.at(AdaLayernorm_backward_attributes::output_names::DBIAS); + if (DBIAS->get_is_virtual() == false) { + infer_scale_bias_tensors(DBIAS); + } + + return {error_code_t::OK, ""}; + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating DAdaLayerNormNode node " << attributes.name); + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations, + std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final { + getLogger() << "[cudnn_frontend] INFO: " << "Building DAdaLayerNormNode operations " << attributes.name + << std::endl; + auto cudnn_ver_error = error_t{error_code_t::GRAPH_NOT_SUPPORTED, "AdaLN bwd requires cuDNN v9.9.0"}; +#if (CUDNN_VERSION >= 90900) + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(90900, cudnn_ver_error); + CUDNN_FRONTEND_UNUSED(operations); + auto adalayernorm_operation = + make_shared_backend_pointer((cudnnBackendDescriptorType_t)CUDNN_BACKEND_OPERATION_NORM_BACKWARD_DESCRIPTOR); + + cudnnBackendNormMode_t cudnn_norm_mode; + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormMode_t::ADA_LAYER_NORM, cudnn_norm_mode)); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_NORM_BWD_MODE, + CUDNN_TYPE_NORM_MODE, + 1, + &cudnn_norm_mode)); + + auto X = attributes.inputs.find(AdaLayernorm_backward_attributes::input_names::X)->second; + auto backend_x = tensors[X->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_NORM_BWD_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_x)); + + auto Mean = attributes.inputs.find(AdaLayernorm_backward_attributes::input_names::MEAN)->second; + auto backend_mean = tensors[Mean->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_NORM_BWD_MEAN_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_mean)); + + auto Inv_variance = attributes.inputs.find(AdaLayernorm_backward_attributes::input_names::INV_VARIANCE)->second; + auto backend_inv_variance = tensors[Inv_variance->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_NORM_BWD_INV_VARIANCE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_inv_variance)); + + auto Dy = attributes.inputs.find(AdaLayernorm_backward_attributes::input_names::DY)->second; + auto backend_dy = tensors[Dy->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_NORM_BWD_DYDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_dy)); + + auto Scale = attributes.inputs.find(AdaLayernorm_backward_attributes::input_names::SCALE)->second; + auto backend_scale = tensors[Scale->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_NORM_BWD_SCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_scale)); + + auto Dx = attributes.outputs.find(AdaLayernorm_backward_attributes::output_names::DX)->second; + auto backend_dx = tensors[Dx->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_NORM_BWD_DXDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_dx)); + + auto Dscale = attributes.outputs.find(AdaLayernorm_backward_attributes::output_names::DSCALE)->second; + auto backend_dscale = tensors[Dscale->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_NORM_BWD_DSCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_dscale)); + + auto Dbias_iter = attributes.outputs.find(AdaLayernorm_backward_attributes::output_names::DBIAS); + if (Dbias_iter != attributes.outputs.end() && Dbias_iter->second->get_is_virtual() == false) { + auto backend_dbias = tensors[Dbias_iter->second->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_NORM_BWD_DBIAS_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_dbias)); + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(adalayernorm_operation->get_backend_descriptor())); + + raw_operations.push_back(adalayernorm_operation); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; +#else + CUDNN_FRONTEND_UNUSED(uids_involved_in_operations); + CUDNN_FRONTEND_UNUSED(operations); + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FRONTEND_UNUSED(tensors); + return cudnn_ver_error; +#endif + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "ADA_LAYER_NORM_BPROP"})"_json); + } +#endif +}; + +} // namespace graph + +} // namespace cudnn_frontend
\ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/batchnorm.h b/deps/cudnn-frontend/include/cudnn_frontend/node/batchnorm.h new file mode 100644 index 0000000..69d232b --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/batchnorm.h @@ -0,0 +1,268 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend { + +namespace graph { +class BatchNormNode : public NodeCRTP<BatchNormNode> { + public: + Batchnorm_attributes attributes; + + BatchNormNode(Batchnorm_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::BATCHNORM; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for batchnorm node " << attributes.name); + + attributes.fill_from_context(context); + + auto X = attributes.inputs[Batchnorm_attributes::input_names::X]; + auto Y = attributes.outputs[Batchnorm_attributes::output_names::Y]; + // Only infer dims and strides if user did not set them + if (Y->get_dim().empty()) { + Y->set_dim(X->get_dim()); + } + if (Y->get_stride().empty()) { + Y->set_stride(X->get_stride()); + } + + // Set channel length tensors + auto const x_tensor_dim = X->get_dim(); + auto infer_per_channel_tensors = [&x_tensor_dim](std::shared_ptr<Tensor_attributes>& T) { + auto tensor_dim = T->get_dim(); + // Only infer dims and strides if user did not set them + if (tensor_dim.empty()) { + tensor_dim.resize(x_tensor_dim.size(), 1); + tensor_dim[1] = x_tensor_dim[1]; + T->set_dim(tensor_dim); + } + if (T->get_stride().empty()) { + auto const& T_dim = T->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(T_dim.size()); + T->set_stride(detail::generate_stride(T_dim, stride_order)); + } + }; + infer_per_channel_tensors(attributes.outputs[Batchnorm_attributes::output_names::MEAN]); + infer_per_channel_tensors(attributes.outputs[Batchnorm_attributes::output_names::INV_VARIANCE]); + + auto has_running_stats = attributes.inputs[Batchnorm_attributes::input_names::PREV_RUNNING_MEAN] || + attributes.inputs[Batchnorm_attributes::input_names::PREV_RUNNING_VAR]; + + if (has_running_stats) { + infer_per_channel_tensors(attributes.outputs[Batchnorm_attributes::output_names::NEXT_RUNNING_MEAN]); + infer_per_channel_tensors(attributes.outputs[Batchnorm_attributes::output_names::NEXT_RUNNING_VAR]); + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations, + std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: Building BatchNormNode operations " << attributes.name << " "); + + // Create operation by directly calling cuDNN backend API + Operation_v8 batchnorm_operation; + + _CUDNN_CHECK_CUDNN_ERROR( + batchnorm_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_NORM_FORWARD_DESCRIPTOR)); + + // Set norm mode to BATCH_NORM + cudnnBackendNormMode_t cudnn_norm_mode; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormMode_t::BATCH_NORM, cudnn_norm_mode)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_MODE, + CUDNN_TYPE_NORM_MODE, + 1, + &cudnn_norm_mode)); + + // Set forward phase to TRAINING + cudnnBackendNormFwdPhase_t cudnn_norm_fwd_phase; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormFwdPhase_t::TRAINING, cudnn_norm_fwd_phase)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_PHASE, + CUDNN_TYPE_NORM_FWD_PHASE, + 1, + &cudnn_norm_fwd_phase)); + + // Set input tensor X + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Batchnorm_attributes::input_names::X); + auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &x_desc)); + + // Set saved mean and inv_variance + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(MEAN, Batchnorm_attributes::output_names::MEAN); + auto mean_desc = tensors.at(MEAN->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_MEAN_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &mean_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(INV_VARIANCE, Batchnorm_attributes::output_names::INV_VARIANCE); + auto inv_var_desc = tensors.at(INV_VARIANCE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_INV_VARIANCE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &inv_var_desc)); + + // Set scale and bias tensors + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SCALE, Batchnorm_attributes::input_names::SCALE); + auto scale_desc = tensors.at(SCALE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_SCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &scale_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(BIAS, Batchnorm_attributes::input_names::BIAS); + auto bias_desc = tensors.at(BIAS->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_BIAS_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &bias_desc)); + + // Set epsilon tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(EPSILON, Batchnorm_attributes::input_names::EPSILON); + auto epsilon_desc = tensors.at(EPSILON->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_EPSILON_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &epsilon_desc)); + + // Check for running stats + bool has_running_stats = true; + auto it = attributes.inputs.find(Batchnorm_attributes::input_names::PREV_RUNNING_MEAN); + if (it == attributes.inputs.end() || it->second == nullptr) { + has_running_stats = false; + } + + if (has_running_stats) { + // Set momentum (exp decay factor) + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(MOMENTUM, Batchnorm_attributes::input_names::MOMENTUM); + auto momentum_desc = tensors.at(MOMENTUM->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_EXP_AVG_FACTOR_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &momentum_desc)); + + // Set prev running mean and var + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(PREV_RUNNING_MEAN, + Batchnorm_attributes::input_names::PREV_RUNNING_MEAN); + auto prev_mean_desc = tensors.at(PREV_RUNNING_MEAN->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_INPUT_RUNNING_MEAN_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &prev_mean_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(PREV_RUNNING_VAR, + Batchnorm_attributes::input_names::PREV_RUNNING_VAR); + auto prev_var_desc = tensors.at(PREV_RUNNING_VAR->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_INPUT_RUNNING_VAR_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &prev_var_desc)); + + // Set next running mean and var + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(NEXT_RUNNING_MEAN, + Batchnorm_attributes::output_names::NEXT_RUNNING_MEAN); + auto next_mean_desc = tensors.at(NEXT_RUNNING_MEAN->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_OUTPUT_RUNNING_MEAN_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &next_mean_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(NEXT_RUNNING_VAR, + Batchnorm_attributes::output_names::NEXT_RUNNING_VAR); + auto next_var_desc = tensors.at(NEXT_RUNNING_VAR->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_OUTPUT_RUNNING_VAR_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &next_var_desc)); + } + + // Set output tensor Y + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(Y, Batchnorm_attributes::output_names::Y); + auto y_desc = tensors.at(Y->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_YDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &y_desc)); + + // Set peer stat tensors if any + if (!attributes.peer_stats.empty()) { + std::vector<cudnnBackendDescriptor_t> peer_stat_descs; + for (auto const& peer_stat : attributes.peer_stats) { + peer_stat_descs.push_back(tensors.at(peer_stat->get_uid())->get_raw_desc()); + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_PEER_STAT_DESCS, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + peer_stat_descs.size(), + peer_stat_descs.data())); + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(batchnorm_operation.get_raw_desc())); + + operations.push_back(std::make_shared<Operation_v8>(std::move(batchnorm_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "BATCHNORM"})"_json); + } +#endif +}; + +} // namespace graph + +} // namespace cudnn_frontend
\ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/batchnorm_inference.h b/deps/cudnn-frontend/include/cudnn_frontend/node/batchnorm_inference.h new file mode 100644 index 0000000..9fb433f --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/batchnorm_inference.h @@ -0,0 +1,156 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend { + +namespace graph { +class BatchnormInferenceNode : public NodeCRTP<BatchnormInferenceNode> { + public: + Batchnorm_inference_attributes attributes; + + BatchnormInferenceNode(Batchnorm_inference_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::BATCHNORM_INFERENCE; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for batchnorm inference node " << attributes.name); + + attributes.fill_from_context(context); + + auto X = attributes.inputs[Batchnorm_inference_attributes::input_names::X]; + auto Y = attributes.outputs[Batchnorm_inference_attributes::output_names::Y]; + // Only infer dims and strides if user did not set them + if (Y->get_dim().empty()) { + Y->set_dim(X->get_dim()); + } + if (Y->get_stride().empty()) { + Y->set_stride(X->get_stride()); + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations, + std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: Building BatchnormInferenceNode operations " << attributes.name << " "); + + // Create operation by directly calling cuDNN backend API + Operation_v8 batchnorm_operation; + + _CUDNN_CHECK_CUDNN_ERROR( + batchnorm_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_NORM_FORWARD_DESCRIPTOR)); + + // Set norm mode to BATCH_NORM + cudnnBackendNormMode_t cudnn_norm_mode; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormMode_t::BATCH_NORM, cudnn_norm_mode)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_MODE, + CUDNN_TYPE_NORM_MODE, + 1, + &cudnn_norm_mode)); + + // Set forward phase to INFERENCE + cudnnBackendNormFwdPhase_t cudnn_norm_fwd_phase; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormFwdPhase_t::INFERENCE, cudnn_norm_fwd_phase)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_PHASE, + CUDNN_TYPE_NORM_FWD_PHASE, + 1, + &cudnn_norm_fwd_phase)); + + // Set input tensor X + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Batchnorm_inference_attributes::input_names::X); + auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &x_desc)); + + // Set mean and inv_variance (as inputs for inference) + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(MEAN, Batchnorm_inference_attributes::input_names::MEAN); + auto mean_desc = tensors.at(MEAN->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_MEAN_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &mean_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(INV_VARIANCE, + Batchnorm_inference_attributes::input_names::INV_VARIANCE); + auto inv_var_desc = tensors.at(INV_VARIANCE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_INV_VARIANCE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &inv_var_desc)); + + // Set scale and bias tensors + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SCALE, Batchnorm_inference_attributes::input_names::SCALE); + auto scale_desc = tensors.at(SCALE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_SCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &scale_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(BIAS, Batchnorm_inference_attributes::input_names::BIAS); + auto bias_desc = tensors.at(BIAS->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_BIAS_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &bias_desc)); + + // Set output tensor Y + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(Y, Batchnorm_inference_attributes::output_names::Y); + auto y_desc = tensors.at(Y->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_YDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &y_desc)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(batchnorm_operation.get_raw_desc())); + + operations.push_back(std::make_shared<Operation_v8>(std::move(batchnorm_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "BATCHNORM_INFERENCE"})"_json); + } +#endif +}; + +} // namespace graph + +} // namespace cudnn_frontend
\ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/block_scale_dequantize.h b/deps/cudnn-frontend/include/cudnn_frontend/node/block_scale_dequantize.h new file mode 100644 index 0000000..bbaa2bf --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/block_scale_dequantize.h @@ -0,0 +1,169 @@ +#pragma once + +#include "../../cudnn_frontend_Logging.h" +#include "../../cudnn_frontend_shim.h" + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend { + +namespace graph { + +class BlockScaleDequantizeNode : public NodeCRTP<BlockScaleDequantizeNode> { + public: + Block_scale_dequantize_attributes attributes; + + BlockScaleDequantizeNode(Block_scale_dequantize_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::BLOCK_SCALE_DEQUANTIZE; + } + + error_t + pre_validate_node() const override final { + getLogger() << "[cudnn_frontend] INFO: " + << "Validating BlockScaleDequantizeNode " << attributes.name << std::endl; + + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.block_size.empty(), error_code_t::ATTRIBUTE_NOT_SET, "Block size not set\n"); + + auto Y = attributes.outputs.at(Block_scale_dequantize_attributes::output_names::Y); + + RETURN_CUDNN_FRONTEND_ERROR_IF(!(Y->get_is_virtual()), + error_code_t::INVALID_VALUE, + "Output tensor of dequantize node should be virtual\n"); + + return {error_code_t::OK, ""}; + } + + error_t + infer_properties_node() override final { + getLogger() << "[cudnn_frontend] INFO: Inferencing properties for BlockScaleDequantizeNode " << attributes.name + << std::endl; + + attributes.fill_from_context(context); + + auto X = attributes.inputs[Block_scale_dequantize_attributes::input_names::X]; + auto scale = attributes.inputs[Block_scale_dequantize_attributes::input_names::scale]; + auto Y = attributes.outputs[Block_scale_dequantize_attributes::output_names::Y]; + + // Only infer dims and strides if user did not set them + if (Y->get_dim().empty()) { + Y->set_dim(X->get_dim()); + } + + if (Y->get_stride().empty()) { + Y->set_stride(X->get_stride()); + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations, + std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final { + getLogger() << "[cudnn_frontend] INFO: " + << "Building BlockScaleDequantizeNode operations " << attributes.name << std::endl; + auto cudnn_ver_error = + error_t{error_code_t::GRAPH_NOT_SUPPORTED, "Block scale dequantize requires cuDNN v9.7.0"}; + +#if (CUDNN_VERSION >= 90700) // TODO: v9.99 is new feature branch; switch to release branch when ready + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(90700, cudnn_ver_error); + CUDNN_FRONTEND_UNUSED(operations); + auto block_scale_dequantize_operation = make_shared_backend_pointer( + (cudnnBackendDescriptorType_t)CUDNN_BACKEND_OPERATION_BLOCK_SCALE_DEQUANTIZE_DESCRIPTOR); + + auto X = attributes.inputs.find(Block_scale_dequantize_attributes::input_names::X)->second; + auto backend_x = tensors[X->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(block_scale_dequantize_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_BLOCK_SCALE_DEQUANTIZE_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_x)); + + auto scale = attributes.inputs.find(Block_scale_dequantize_attributes::input_names::scale)->second; + auto backend_scale = tensors[scale->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(block_scale_dequantize_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_BLOCK_SCALE_DEQUANTIZE_SCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_scale)); + + auto Y = attributes.outputs.find(Block_scale_dequantize_attributes::output_names::Y)->second; + auto backend_y = tensors[Y->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(block_scale_dequantize_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_BLOCK_SCALE_DEQUANTIZE_YDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_y)); + + cudnnDataType_t cudnn_data_type; + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.compute_data_type, cudnn_data_type)); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(block_scale_dequantize_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_BLOCK_SCALE_DEQUANTIZE_MATH_PREC, + CUDNN_TYPE_DATA_TYPE, + 1, + &cudnn_data_type)); + + const int32_t* block_size = attributes.block_size.data(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(block_scale_dequantize_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_BLOCK_SCALE_DEQUANTIZE_BLOCK_SIZE, + CUDNN_TYPE_INT32, + attributes.block_size.size(), + block_size)); + +#if (CUDNN_VERSION >= 91400) + if (detail::get_backend_version() >= 91400) { + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(block_scale_dequantize_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_BLOCK_SCALE_DEQUANTIZE_NEG_SCALE, + CUDNN_TYPE_BOOLEAN, + 1, + &attributes.is_negative_scale)); + } +#endif + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(block_scale_dequantize_operation->get_backend_descriptor())); + + raw_operations.push_back(block_scale_dequantize_operation); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; +#else + CUDNN_FRONTEND_UNUSED(uids_involved_in_operations); + CUDNN_FRONTEND_UNUSED(operations); + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FRONTEND_UNUSED(tensors); + return cudnn_ver_error; +#endif + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "BLOCK_SCALE_DEQUANTIZE"})"_json); + } +#endif +}; + +inline void +INode::block_scale_dequantize(std::shared_ptr<Tensor_attributes> x, + std::shared_ptr<Tensor_attributes> scale, + Block_scale_dequantize_attributes attributes, + std::shared_ptr<Tensor_attributes> y) { + attributes.inputs[Block_scale_dequantize_attributes::input_names::X] = x; + attributes.inputs[Block_scale_dequantize_attributes::input_names::scale] = scale; + attributes.outputs[Block_scale_dequantize_attributes::output_names::Y] = y; + sub_nodes.emplace_back(std::make_unique<BlockScaleDequantizeNode>(std::move(attributes), context)); +} + +} // namespace graph + +} // namespace cudnn_frontend
\ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/block_scale_quantize.h b/deps/cudnn-frontend/include/cudnn_frontend/node/block_scale_quantize.h new file mode 100644 index 0000000..494ea14 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/block_scale_quantize.h @@ -0,0 +1,205 @@ +#pragma once + +#include "../../cudnn_frontend_Logging.h" +#include "../../cudnn_frontend_shim.h" + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend { + +namespace graph { + +class BlockScaleQuantizeNode : public NodeCRTP<BlockScaleQuantizeNode> { + public: + Block_scale_quantize_attributes attributes; + + BlockScaleQuantizeNode(Block_scale_quantize_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::BLOCK_SCALE_QUANTIZE; + } + + error_t + pre_validate_node() const override final { + getLogger() << "[cudnn_frontend] INFO: " << "Validating BlockScaleQuantizeNode " << attributes.name + << std::endl; + + RETURN_CUDNN_FRONTEND_ERROR_IF( + !attributes.block_size.has_value(), error_code_t::ATTRIBUTE_NOT_SET, "Block size not set."); + + return {error_code_t::OK, ""}; + } + + error_t + infer_properties_node() override final { + getLogger() << "[cudnn_frontend] INFO: Inferencing properties for BlockScaleQuantizeNode " << attributes.name + << std::endl; + + attributes.fill_from_context(context); + + auto X = attributes.inputs[Block_scale_quantize_attributes::input_names::X]; + auto Y = attributes.outputs[Block_scale_quantize_attributes::output_names::Y]; + auto scale = attributes.outputs[Block_scale_quantize_attributes::output_names::scale]; + + // Block scale quantize requires the block scale axis to be packed + auto infer_strides_transposed = [&X](std::shared_ptr<Tensor_attributes>& T, + std::optional<int64_t> const& axis) { + auto const& dim = T->get_dim(); + auto const& X_dim = X->get_dim(); + auto const& X_stride = X->get_stride(); + + std::vector<int64_t> indices(X_stride.size()); + std::iota(indices.begin(), indices.end(), 0); + // Sort indices based on stride values in descending order + std::sort(indices.begin(), indices.end(), [&X_dim, &X_stride](int64_t i, int64_t j) { + // Prioritize singleton dimensions + if (X_stride[i] == X_stride[j]) { + return (X_dim[i] == 1) && (X_dim[j] != 1); + } + return X_stride[i] < X_stride[j]; + }); + if (axis) { + // Rotate left until the axis is the packed dim + std::rotate(indices.begin(), std::find(indices.begin(), indices.end(), axis.value()), indices.end()); + } + std::vector<int64_t> stride_order(X_stride.size()); + for (size_t i = 0; i < indices.size(); ++i) { + stride_order[indices[i]] = i; + } + T->set_stride(detail::generate_stride(dim, stride_order)); + }; + + // Only infer dims and strides if user did not set them + if (Y->get_dim().empty()) { + Y->set_dim(X->get_dim()); + } + if (Y->get_stride().empty()) { + if (attributes.transpose) { + infer_strides_transposed(Y, attributes.axis); + } else { + Y->set_stride(X->get_stride()); + } + } + + // Only infer dims and strides if user did not set them + if (scale->get_dim().empty()) { + auto scale_dim = X->get_dim(); + if (attributes.axis) { + scale_dim[attributes.axis.value()] /= attributes.block_size.value(); + } else { + scale_dim.back() /= attributes.block_size.value(); + } + scale->set_dim(scale_dim); + } + if (scale->get_stride().empty()) { + if (attributes.transpose) { + infer_strides_transposed(scale, attributes.axis); + } else { + auto const& scale_dim = scale->get_dim(); + std::vector<int64_t> stride_order; + CHECK_CUDNN_FRONTEND_ERROR( + detail::generate_stride_order_preserving_format(X->get_stride(), scale_dim.size(), stride_order)); + scale->set_stride(detail::generate_stride(scale_dim, stride_order)); + } + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations, + std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final { + getLogger() << "[cudnn_frontend] INFO: " << "Building BlockScaleQuantizeNode operations " << attributes.name + << std::endl; + auto cudnn_ver_error = error_t{error_code_t::GRAPH_NOT_SUPPORTED, "Block scale quantize requires cuDNN v9.7.0"}; + +#if (CUDNN_VERSION >= 90700) // TODO: v9.99 is new feature branch; switch to release branch when ready + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(90700, cudnn_ver_error); + CUDNN_FRONTEND_UNUSED(operations); + auto block_scale_quantize_operation = make_shared_backend_pointer( + (cudnnBackendDescriptorType_t)CUDNN_BACKEND_OPERATION_BLOCK_SCALE_QUANTIZE_DESCRIPTOR); + + auto X = attributes.inputs.find(Block_scale_quantize_attributes::input_names::X)->second; + auto backend_x = tensors[X->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(block_scale_quantize_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_BLOCK_SCALE_QUANTIZE_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_x)); + + auto Y = attributes.outputs.find(Block_scale_quantize_attributes::output_names::Y)->second; + auto backend_y = tensors[Y->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(block_scale_quantize_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_BLOCK_SCALE_QUANTIZE_YDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_y)); + + auto scale = attributes.outputs.find(Block_scale_quantize_attributes::output_names::scale)->second; + auto backend_scale = tensors[scale->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(block_scale_quantize_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_BLOCK_SCALE_QUANTIZE_SCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_scale)); + + cudnnDataType_t cudnn_data_type; + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.compute_data_type, cudnn_data_type)); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(block_scale_quantize_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_BLOCK_SCALE_QUANTIZE_MATH_PREC, + CUDNN_TYPE_DATA_TYPE, + 1, + &cudnn_data_type)); + + int32_t block_size = attributes.block_size.value(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(block_scale_quantize_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_BLOCK_SCALE_QUANTIZE_BLOCK_SIZE, + CUDNN_TYPE_INT32, + 1, + &block_size)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(block_scale_quantize_operation->get_backend_descriptor())); + + raw_operations.push_back(block_scale_quantize_operation); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; +#else + CUDNN_FRONTEND_UNUSED(uids_involved_in_operations); + CUDNN_FRONTEND_UNUSED(operations); + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FRONTEND_UNUSED(tensors); + return cudnn_ver_error; +#endif + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "BLOCK_SCALE_QUANTIZE"})"_json); + } +#endif +}; + +inline void +INode::block_scale_quantize(std::shared_ptr<Tensor_attributes> x, + Block_scale_quantize_attributes attributes, + std::shared_ptr<Tensor_attributes> y, + std::shared_ptr<Tensor_attributes> scale) { + attributes.inputs[Block_scale_quantize_attributes::input_names::X] = x; + attributes.outputs[Block_scale_quantize_attributes::output_names::Y] = y; + attributes.outputs[Block_scale_quantize_attributes::output_names::scale] = scale; + sub_nodes.emplace_back(std::make_unique<BlockScaleQuantizeNode>(std::move(attributes), context)); +} + +} // namespace graph + +} // namespace cudnn_frontend diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/bn_finalize.h b/deps/cudnn-frontend/include/cudnn_frontend/node/bn_finalize.h new file mode 100644 index 0000000..99d3ec0 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/bn_finalize.h @@ -0,0 +1,263 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend { + +namespace graph { + +class BatchNormFinalizeNode : public NodeCRTP<BatchNormFinalizeNode> { + public: + BN_finalize_attributes attributes; + + BatchNormFinalizeNode(BN_finalize_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::BN_FINALIZE; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO:Inferencing properties for batchnorm finalize node " << attributes.name); + + attributes.fill_from_context(context); + + auto SUM = attributes.inputs[BN_finalize_attributes::input_names::SUM]; + auto const sum_tensor_dim = SUM->get_dim(); + + // Set channel length tensors + auto infer_per_channel_tensors = [&sum_tensor_dim](std::shared_ptr<Tensor_attributes>& T) { + auto tensor_dim = T->get_dim(); + // Only infer dims and strides if user did not set them + if (tensor_dim.empty()) { + tensor_dim = sum_tensor_dim; + T->set_dim(tensor_dim); + } + if (T->get_stride().empty()) { + auto const& T_dim = T->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(T_dim.size()); + T->set_stride(detail::generate_stride(T_dim, stride_order)); + } + }; + infer_per_channel_tensors(attributes.outputs[BN_finalize_attributes::output_names::EQ_BIAS]); + infer_per_channel_tensors(attributes.outputs[BN_finalize_attributes::output_names::EQ_SCALE]); + infer_per_channel_tensors(attributes.outputs[BN_finalize_attributes::output_names::MEAN]); + infer_per_channel_tensors(attributes.outputs[BN_finalize_attributes::output_names::INV_VARIANCE]); + infer_per_channel_tensors(attributes.outputs[BN_finalize_attributes::output_names::NEXT_RUNNING_MEAN]); + infer_per_channel_tensors(attributes.outputs[BN_finalize_attributes::output_names::NEXT_RUNNING_VAR]); + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations, + std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: " << "Building BatchNormFinalizeNode operations " << attributes.name << " "); + + // Create operation by directly calling cuDNN backend API + Operation_v8 bn_finalize_operation; + + _CUDNN_CHECK_CUDNN_ERROR(bn_finalize_operation.initialize_managed_backend_pointer( + CUDNN_BACKEND_OPERATION_BN_FINALIZE_STATISTICS_DESCRIPTOR)); + + // Set BN finalize mode + cudnnBnFinalizeStatsMode_t bn_finalize_mode = CUDNN_BN_FINALIZE_STATISTICS_TRAINING; + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_FINALIZE_STATS_MODE, + CUDNN_TYPE_BN_FINALIZE_STATS_MODE, + 1, + &bn_finalize_mode)); + + // Set compute type (math precision) + cudnnDataType_t compute_type = CUDNN_DATA_FLOAT; + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_FINALIZE_MATH_PREC, + CUDNN_TYPE_DATA_TYPE, + 1, + &compute_type)); + + // Set SUM input tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SUM, BN_finalize_attributes::input_names::SUM); + auto sum_desc = tensors.at(SUM->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_FINALIZE_Y_SUM_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &sum_desc)); + + // Set SQ_SUM input tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SQ_SUM, BN_finalize_attributes::input_names::SQ_SUM); + auto sq_sum_desc = tensors.at(SQ_SUM->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_FINALIZE_Y_SQ_SUM_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &sq_sum_desc)); + + // Set SCALE input tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SCALE, BN_finalize_attributes::input_names::SCALE); + auto scale_desc = tensors.at(SCALE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_FINALIZE_SCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &scale_desc)); + + // Set BIAS input tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(BIAS, BN_finalize_attributes::input_names::BIAS); + auto bias_desc = tensors.at(BIAS->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_FINALIZE_BIAS_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &bias_desc)); + + // Set EQ_SCALE output tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(EQ_SCALE, BN_finalize_attributes::output_names::EQ_SCALE); + auto eq_scale_desc = tensors.at(EQ_SCALE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_FINALIZE_EQ_SCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &eq_scale_desc)); + + // Set EQ_BIAS output tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(EQ_BIAS, BN_finalize_attributes::output_names::EQ_BIAS); + auto eq_bias_desc = tensors.at(EQ_BIAS->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_FINALIZE_EQ_BIAS_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &eq_bias_desc)); + + // Set PREV_RUNNING_MEAN input tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(PREV_RUNNING_MEAN, + BN_finalize_attributes::input_names::PREV_RUNNING_MEAN); + auto prev_running_mean_desc = tensors.at(PREV_RUNNING_MEAN->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_FINALIZE_PREV_RUNNING_MEAN_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &prev_running_mean_desc)); + + // Set PREV_RUNNING_VAR input tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(PREV_RUNNING_VAR, + BN_finalize_attributes::input_names::PREV_RUNNING_VAR); + auto prev_running_var_desc = tensors.at(PREV_RUNNING_VAR->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_FINALIZE_PREV_RUNNING_VAR_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &prev_running_var_desc)); + + // Set NEXT_RUNNING_MEAN output tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(NEXT_RUNNING_MEAN, + BN_finalize_attributes::output_names::NEXT_RUNNING_MEAN); + auto next_running_mean_desc = tensors.at(NEXT_RUNNING_MEAN->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_FINALIZE_UPDATED_RUNNING_MEAN_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &next_running_mean_desc)); + + // Set NEXT_RUNNING_VAR output tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(NEXT_RUNNING_VAR, + BN_finalize_attributes::output_names::NEXT_RUNNING_VAR); + auto next_running_var_desc = tensors.at(NEXT_RUNNING_VAR->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_FINALIZE_UPDATED_RUNNING_VAR_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &next_running_var_desc)); + + // Set MEAN output tensor (saved mean) + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(MEAN, BN_finalize_attributes::output_names::MEAN); + auto mean_desc = tensors.at(MEAN->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_FINALIZE_SAVED_MEAN_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &mean_desc)); + + // Set INV_VARIANCE output tensor (saved inv std) + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(INV_VARIANCE, BN_finalize_attributes::output_names::INV_VARIANCE); + auto inv_var_desc = tensors.at(INV_VARIANCE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_FINALIZE_SAVED_INV_STD_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &inv_var_desc)); + + // Set EPSILON tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(EPSILON, BN_finalize_attributes::input_names::EPSILON); + auto epsilon_desc = tensors.at(EPSILON->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_FINALIZE_EPSILON_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &epsilon_desc)); + + // Set MOMENTUM tensor (exp average factor) + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(MOMENTUM, BN_finalize_attributes::input_names::MOMENTUM); + auto momentum_desc = tensors.at(MOMENTUM->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_FINALIZE_EXP_AVERATE_FACTOR_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &momentum_desc)); + + // Set ACCUM_COUNT tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(ACCUM_COUNT, BN_finalize_attributes::input_names::ACCUM_COUNT); + auto accum_count_desc = tensors.at(ACCUM_COUNT->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_FINALIZE_ACCUM_COUNT_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &accum_count_desc)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(bn_finalize_operation.get_raw_desc())); + + operations.push_back(std::make_shared<Operation_v8>(std::move(bn_finalize_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "BN_FINALIZE"})"_json); + } +#endif +}; + +} // namespace graph + +} // namespace cudnn_frontend
\ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/concatenate.h b/deps/cudnn-frontend/include/cudnn_frontend/node/concatenate.h new file mode 100644 index 0000000..5f62589 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/concatenate.h @@ -0,0 +1,163 @@ +#pragma once + +#include "../../cudnn_frontend_Logging.h" +#include "../../cudnn_frontend_shim.h" + +#include "../graph_helpers.h" +#include "../node_interface.h" +#include <string> + +namespace cudnn_frontend { + +namespace graph { + +class ConcatenateNode : public NodeCRTP<ConcatenateNode> { + public: + Concatenate_attributes attributes; + + ConcatenateNode(Concatenate_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::CONCATENATE; + } + + error_t + pre_validate_node() const override final { + getLogger() << "[cudnn_frontend] INFO: " << "Validating ConcatenateNode " << attributes.name << std::endl; + + RETURN_CUDNN_FRONTEND_ERROR_IF(!attributes.axis.has_value(), error_code_t::ATTRIBUTE_NOT_SET, "Axis not set\n"); + + auto X = attributes.inputs; + + RETURN_CUDNN_FRONTEND_ERROR_IF( + (X.size() == 0), error_code_t::INVALID_VALUE, "Input size of the concatenate node cannot be zero\n"); + + return {error_code_t::OK, ""}; + } + + error_t + infer_properties_node() override final { + getLogger() << "[cudnn_frontend] INFO: Inferring properties for ConcatenateNode " << attributes.name + << std::endl; + + attributes.fill_from_context(context); + + auto Y = attributes.outputs[Concatenate_attributes::output_names::Y]; + + // Infer dims and strides only if user did not set them + int64_t dim_sum = 0; + for (const auto& input : attributes.inputs) { + dim_sum += input->get_dim()[attributes.axis.value()]; + } + + auto X = attributes.inputs[0]; + auto dims = X->get_dim(); + dims[attributes.axis.value()] = dim_sum; + + if (Y->get_dim().empty()) { + Y->set_dim(dims); + Y->set_dim(dims); + } + + if (Y->get_stride().empty()) { + std::vector<int64_t> stride_order; + CHECK_CUDNN_FRONTEND_ERROR( + detail::generate_stride_order_preserving_format(X->get_stride(), dims.size(), stride_order)); + Y->set_stride(detail::generate_stride(dims, stride_order)); + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations, + std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final { + getLogger() << "[cudnn_frontend] INFO: " << "Building ConcatenateNode operations " << attributes.name + << std::endl; + auto cudnn_ver_error = error_t{error_code_t::GRAPH_NOT_SUPPORTED, "Concatenate requires cuDNN v9.7.0"}; + +#if (CUDNN_VERSION >= 90700) + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(90700, cudnn_ver_error); + CUDNN_FRONTEND_UNUSED(operations); + auto concatenate_operation = make_shared_backend_pointer(CUDNN_BACKEND_OPERATION_CONCAT_DESCRIPTOR); + + std::vector<void*> backend_x(attributes.inputs.size()); + size_t index = 0; + for (const auto& input : attributes.inputs) { + backend_x[index] = tensors[input->get_uid()]->get_desc()->get_backend_descriptor(); + index++; + } + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(concatenate_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_CONCAT_INPUT_DESCS, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + attributes.inputs.size(), + backend_x.data())); + + auto Y = attributes.outputs.find(Concatenate_attributes::output_names::Y)->second; + auto backend_y = tensors[Y->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(concatenate_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_CONCAT_OUTPUT_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_y)); + + int64_t axis = attributes.axis.value(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(concatenate_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_CONCAT_AXIS, + CUDNN_TYPE_INT64, + 1, + &axis)); + + if (attributes.in_place_index.has_value()) { + int64_t in_place_index = attributes.in_place_index.value(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(concatenate_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_CONCAT_INPLACE_INDEX, + CUDNN_TYPE_INT64, + 1, + &in_place_index)); + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(concatenate_operation->get_backend_descriptor())); + + raw_operations.push_back(concatenate_operation); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; +#else + CUDNN_FRONTEND_UNUSED(uids_involved_in_operations); + CUDNN_FRONTEND_UNUSED(operations); + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FRONTEND_UNUSED(tensors); + return cudnn_ver_error; +#endif + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "CONCATENATE"})"_json); + } +#endif +}; + +inline void +INode::concatenate(std::vector<std::shared_ptr<Tensor_attributes>> x, + Concatenate_attributes attributes, + std::shared_ptr<Tensor_attributes> y) { + for (auto& element : x) { + attributes.inputs.push_back(element); + } + attributes.outputs[Concatenate_attributes::output_names::Y] = y; + sub_nodes.emplace_back(std::make_unique<ConcatenateNode>(std::move(attributes), context)); +} + +} // namespace graph + +} // namespace cudnn_frontend diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/conv_dgrad.h b/deps/cudnn-frontend/include/cudnn_frontend/node/conv_dgrad.h new file mode 100644 index 0000000..a2e2e4e --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/conv_dgrad.h @@ -0,0 +1,205 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend::graph { + +class DgradNode : public NodeCRTP<DgradNode> { + public: + Conv_dgrad_attributes attributes; + + DgradNode(Conv_dgrad_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::DGRAD; + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating Node Type::DGRAD " << attributes.name); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.get_pre_padding().empty(), error_code_t::ATTRIBUTE_NOT_SET, "Pre padding not set."); + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.get_post_padding().empty(), error_code_t::ATTRIBUTE_NOT_SET, "Post padding not set."); + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.get_stride().empty(), error_code_t::ATTRIBUTE_NOT_SET, "Conv strides not set."); + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.get_dilation().empty(), error_code_t::ATTRIBUTE_NOT_SET, "Conv dilation not set."); + + return {error_code_t::OK, ""}; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for dgrad node " << attributes.name); + + attributes.fill_from_context(context); + + // TODO: Only inferrencing from (X, DY) -> DW works today. + auto DX = attributes.outputs.find(Conv_dgrad_attributes::output_names::DX)->second; + auto W = attributes.inputs.find(Conv_dgrad_attributes::input_names::W)->second; + auto DY = attributes.inputs.find(Conv_dgrad_attributes::input_names::DY)->second; + + auto const w_tensor_dim = W->get_dim(); + auto const dy_tensor_dim = DY->get_dim(); + auto dx_tensor_dim = DX->get_dim(); + + RETURN_CUDNN_FRONTEND_ERROR_IF(DX->get_dim().empty(), + error_code_t::ATTRIBUTE_NOT_SET, + "For dgrad node, output dimension inferencing is not possible."); + + // No dim inferencing as inverse mapping from DY, W to DX is not unique. + // Only infer strides if user did not set them + if (DX->get_stride().empty()) { + auto const& DX_dim = DX->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(DX_dim.size()); + DX->set_stride(detail::generate_stride(DX_dim, stride_order)); + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations, + std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: Building DgradNode operations " << attributes.name << " "); + + // Create dgrad descriptor by directly calling cuDNN backend API + ConvDesc_v8 dgrad_descriptor; + int64_t const spatial_dim_count = attributes.get_pre_padding().size(); + + _CUDNN_CHECK_CUDNN_ERROR( + dgrad_descriptor.initialize_managed_backend_pointer(CUDNN_BACKEND_CONVOLUTION_DESCRIPTOR)); + + cudnnDataType_t cudnn_data_type; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.compute_data_type, cudnn_data_type)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dgrad_descriptor.get_raw_desc(), + CUDNN_ATTR_CONVOLUTION_COMP_TYPE, + CUDNN_TYPE_DATA_TYPE, + 1, + &cudnn_data_type)); + + cudnnConvolutionMode_t mode = detail::convert_to_cudnn_type(attributes.math_mode); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute( + dgrad_descriptor.get_raw_desc(), CUDNN_ATTR_CONVOLUTION_CONV_MODE, CUDNN_TYPE_CONVOLUTION_MODE, 1, &mode)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dgrad_descriptor.get_raw_desc(), + CUDNN_ATTR_CONVOLUTION_SPATIAL_DIMS, + CUDNN_TYPE_INT64, + 1, + &spatial_dim_count)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dgrad_descriptor.get_raw_desc(), + CUDNN_ATTR_CONVOLUTION_PRE_PADDINGS, + CUDNN_TYPE_INT64, + spatial_dim_count, + attributes.get_pre_padding().data())); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dgrad_descriptor.get_raw_desc(), + CUDNN_ATTR_CONVOLUTION_POST_PADDINGS, + CUDNN_TYPE_INT64, + spatial_dim_count, + attributes.get_post_padding().data())); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dgrad_descriptor.get_raw_desc(), + CUDNN_ATTR_CONVOLUTION_DILATIONS, + CUDNN_TYPE_INT64, + spatial_dim_count, + attributes.get_dilation().data())); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dgrad_descriptor.get_raw_desc(), + CUDNN_ATTR_CONVOLUTION_FILTER_STRIDES, + CUDNN_TYPE_INT64, + spatial_dim_count, + attributes.get_stride().data())); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(dgrad_descriptor.get_raw_desc())); + CUDNN_FE_LOG_LABEL_ENDL(dgrad_descriptor); + + // Create operation by directly calling cuDNN backend API + Operation_v8 dgrad_operation; + + _CUDNN_CHECK_CUDNN_ERROR(dgrad_operation.initialize_managed_backend_pointer( + CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_DATA_DESCRIPTOR)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DX, Conv_dgrad_attributes::output_names::DX); + auto dx_desc = tensors.at(DX->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dgrad_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_DATA_DX, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dx_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(W, Conv_dgrad_attributes::input_names::W); + auto w_desc = tensors.at(W->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dgrad_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_DATA_W, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &w_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(DY, Conv_dgrad_attributes::input_names::DY); + auto dy_desc = tensors.at(DY->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dgrad_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_DATA_DY, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dy_desc)); + + auto conv_desc_ptr = dgrad_descriptor.get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dgrad_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_DATA_CONV_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &conv_desc_ptr)); + + float alpha = 1.0f; + float beta = 0.0f; + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dgrad_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_DATA_ALPHA, + CUDNN_TYPE_FLOAT, + 1, + &alpha)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dgrad_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_DATA_BETA, + CUDNN_TYPE_FLOAT, + 1, + &beta)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(dgrad_operation.get_raw_desc())); + + operations.push_back(std::make_shared<Operation_v8>(std::move(dgrad_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "CONV_DGRAD"})"_json); + } +#endif +}; + +} // namespace cudnn_frontend::graph
\ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/conv_fprop.h b/deps/cudnn-frontend/include/cudnn_frontend/node/conv_fprop.h new file mode 100644 index 0000000..34acf55 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/conv_fprop.h @@ -0,0 +1,254 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend::graph { +class ConvolutionNode : public NodeCRTP<ConvolutionNode> { + public: + Conv_fprop_attributes attributes; + + ConvolutionNode(Conv_fprop_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::CONVOLUTION; + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating Node Type::CONVOLUTION " << attributes.name); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.get_pre_padding().empty(), error_code_t::ATTRIBUTE_NOT_SET, "Pre padding not set."); + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.get_post_padding().empty(), error_code_t::ATTRIBUTE_NOT_SET, "Post padding not set."); + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.get_stride().empty(), error_code_t::ATTRIBUTE_NOT_SET, "Conv strides not set."); + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.get_dilation().empty(), error_code_t::ATTRIBUTE_NOT_SET, "Conv dilation not set."); + + // Implicit GEMM kernels compute B_offset as int32. When the product of all filter + // dimensions (K * C * spatial...) exceeds INT32_MAX the offset overflows, causing IMA or + // silent NaN. Conservative upper bound — exact check requires kernel tiling params not + // available at graph-validate time. Applies to 2D (4D filter) and 3D (5D filter) fprop. + auto W_it = attributes.inputs.find(Conv_fprop_attributes::input_names::W); + if (W_it != attributes.inputs.end() && W_it->second) { + auto const& w_dim = W_it->second->get_dim(); + if (w_dim.size() == 4 || w_dim.size() == 5) { + int64_t filter_elements = 1; + for (auto d : w_dim) filter_elements *= d; + RETURN_CUDNN_FRONTEND_ERROR_IF( + filter_elements > static_cast<int64_t>(std::numeric_limits<int32_t>::max()), + error_code_t::GRAPH_NOT_SUPPORTED, + "Conv filter total elements exceed INT32_MAX."); + } + } + + return {error_code_t::OK, ""}; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for conv node " << attributes.name); + + attributes.fill_from_context(context); + + // TODO: Only inferrencing from (X, W) -> Y works today. + auto& X = attributes.inputs.find(Conv_fprop_attributes::input_names::X)->second; + auto& W = attributes.inputs.find(Conv_fprop_attributes::input_names::W)->second; + auto& Y = attributes.outputs.find(Conv_fprop_attributes::output_names::Y)->second; + + auto const x_tensor_dim = X->get_dim(); + auto const w_tensor_dim = W->get_dim(); + auto y_tensor_dim = Y->get_dim(); + + // Only infer dims and strides if user did not set them + if (y_tensor_dim.empty()) { + y_tensor_dim.resize(x_tensor_dim.size()); + auto const& pre_padding = attributes.get_pre_padding(); + auto const& post_padding = attributes.get_post_padding(); + auto const& stride = attributes.get_stride(); + auto const& dilation = attributes.get_dilation(); + // N + y_tensor_dim[0] = x_tensor_dim[0]; + // PQ + for (size_t dim = 2; dim < x_tensor_dim.size(); ++dim) { + y_tensor_dim[dim] = 1 + (x_tensor_dim[dim] - dilation[dim - 2] * (w_tensor_dim[dim] - 1) - 1 + + pre_padding[dim - 2] + post_padding[dim - 2]) / + stride[dim - 2]; + } + // K + y_tensor_dim[1] = w_tensor_dim[0]; + Y->set_dim(y_tensor_dim); + } + if (Y->get_stride().empty()) { + auto const& Y_dim = Y->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(Y_dim.size()); + Y->set_stride(detail::generate_stride(Y_dim, stride_order)); + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations, + std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: Building ConvolutionNode operations " << attributes.name << " "); + + // Create convolution descriptor by directly calling cuDNN backend API + ConvDesc_v8 convolution_descriptor; + int64_t const spatial_dim_count = attributes.get_pre_padding().size(); + + _CUDNN_CHECK_CUDNN_ERROR( + convolution_descriptor.initialize_managed_backend_pointer(CUDNN_BACKEND_CONVOLUTION_DESCRIPTOR)); + + // Set compute type + cudnnDataType_t cudnn_data_type; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.compute_data_type, cudnn_data_type)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_descriptor.get_raw_desc(), + CUDNN_ATTR_CONVOLUTION_COMP_TYPE, + CUDNN_TYPE_DATA_TYPE, + 1, + &cudnn_data_type)); + + // Set convolution mode + cudnnConvolutionMode_t mode = detail::convert_to_cudnn_type(attributes.math_mode); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_descriptor.get_raw_desc(), + CUDNN_ATTR_CONVOLUTION_CONV_MODE, + CUDNN_TYPE_CONVOLUTION_MODE, + 1, + &mode)); + + // Set spatial dimensions + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_descriptor.get_raw_desc(), + CUDNN_ATTR_CONVOLUTION_SPATIAL_DIMS, + CUDNN_TYPE_INT64, + 1, + &spatial_dim_count)); + + // Set pre-padding + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_descriptor.get_raw_desc(), + CUDNN_ATTR_CONVOLUTION_PRE_PADDINGS, + CUDNN_TYPE_INT64, + spatial_dim_count, + attributes.get_pre_padding().data())); + + // Set post-padding + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_descriptor.get_raw_desc(), + CUDNN_ATTR_CONVOLUTION_POST_PADDINGS, + CUDNN_TYPE_INT64, + spatial_dim_count, + attributes.get_post_padding().data())); + + // Set dilation + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_descriptor.get_raw_desc(), + CUDNN_ATTR_CONVOLUTION_DILATIONS, + CUDNN_TYPE_INT64, + spatial_dim_count, + attributes.get_dilation().data())); + + // Set strides + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_descriptor.get_raw_desc(), + CUDNN_ATTR_CONVOLUTION_FILTER_STRIDES, + CUDNN_TYPE_INT64, + spatial_dim_count, + attributes.get_stride().data())); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(convolution_descriptor.get_raw_desc())); + CUDNN_FE_LOG_LABEL_ENDL(convolution_descriptor); + + // Create operation by directly calling cuDNN backend API + Operation_v8 convolution_operation; + + _CUDNN_CHECK_CUDNN_ERROR(convolution_operation.initialize_managed_backend_pointer( + CUDNN_BACKEND_OPERATION_CONVOLUTION_FORWARD_DESCRIPTOR)); + + // Set input tensor X + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Conv_fprop_attributes::input_names::X); + auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_X, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &x_desc)); + + // Set weight tensor W + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(W, Conv_fprop_attributes::input_names::W); + auto w_desc = tensors.at(W->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_W, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &w_desc)); + + // Set output tensor Y + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(Y, Conv_fprop_attributes::output_names::Y); + auto y_desc = tensors.at(Y->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_Y, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &y_desc)); + + // Set convolution descriptor + auto conv_desc_ptr = convolution_descriptor.get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_CONV_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &conv_desc_ptr)); + + // Set alpha and beta + float alpha = 1.0f; + float beta = 0.0f; + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_ALPHA, + CUDNN_TYPE_FLOAT, + 1, + &alpha)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_BETA, + CUDNN_TYPE_FLOAT, + 1, + &beta)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(convolution_operation.get_raw_desc())); + + operations.push_back(std::make_shared<Operation_v8>(std::move(convolution_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"({"tag": "CONV_FPROP"})"_json); + } +#endif +}; + +} // namespace cudnn_frontend::graph
\ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/conv_wgrad.h b/deps/cudnn-frontend/include/cudnn_frontend/node/conv_wgrad.h new file mode 100644 index 0000000..2f9b478 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/conv_wgrad.h @@ -0,0 +1,201 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend::graph { + +class WgradNode : public NodeCRTP<WgradNode> { + public: + Conv_wgrad_attributes attributes; + + WgradNode(Conv_wgrad_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::WGRAD; + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating Node Type::WGRAD " << attributes.name); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.get_pre_padding().empty(), error_code_t::ATTRIBUTE_NOT_SET, "Pre padding not set."); + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.get_post_padding().empty(), error_code_t::ATTRIBUTE_NOT_SET, "Post padding not set."); + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.get_stride().empty(), error_code_t::ATTRIBUTE_NOT_SET, "Conv strides not set."); + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.get_dilation().empty(), error_code_t::ATTRIBUTE_NOT_SET, "Conv dilation not set."); + + return {error_code_t::OK, ""}; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for conv node " << attributes.name); + + attributes.fill_from_context(context); + + // TODO: Only inferrencing from (X, DY) -> DW works today. + auto X = attributes.inputs[Conv_wgrad_attributes::input_names::X]; + auto DW = attributes.outputs[Conv_wgrad_attributes::output_names::DW]; + auto DY = attributes.inputs[Conv_wgrad_attributes::input_names::DY]; + + auto const x_tensor_dim = X->get_dim(); + auto const dy_tensor_dim = DY->get_dim(); + auto dw_tensor_dim = DW->get_dim(); + + // No dim inferencing as inverse mapping from DY, X to DX is not unique. + // Only infer strides if user did not set them + if (DW->get_stride().empty()) { + auto const& DW_dim = DW->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(DW_dim.size()); + DW->set_stride(detail::generate_stride(DW_dim, stride_order)); + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations, + std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: Building WgradNode operations " << attributes.name << " "); + + // Create wgrad descriptor by directly calling cuDNN backend API + ConvDesc_v8 wgrad_descriptor; + int64_t const spatial_dim_count = attributes.get_pre_padding().size(); + + _CUDNN_CHECK_CUDNN_ERROR( + wgrad_descriptor.initialize_managed_backend_pointer(CUDNN_BACKEND_CONVOLUTION_DESCRIPTOR)); + + cudnnDataType_t cudnn_data_type; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.compute_data_type, cudnn_data_type)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(wgrad_descriptor.get_raw_desc(), + CUDNN_ATTR_CONVOLUTION_COMP_TYPE, + CUDNN_TYPE_DATA_TYPE, + 1, + &cudnn_data_type)); + + cudnnConvolutionMode_t mode = detail::convert_to_cudnn_type(attributes.math_mode); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute( + wgrad_descriptor.get_raw_desc(), CUDNN_ATTR_CONVOLUTION_CONV_MODE, CUDNN_TYPE_CONVOLUTION_MODE, 1, &mode)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(wgrad_descriptor.get_raw_desc(), + CUDNN_ATTR_CONVOLUTION_SPATIAL_DIMS, + CUDNN_TYPE_INT64, + 1, + &spatial_dim_count)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(wgrad_descriptor.get_raw_desc(), + CUDNN_ATTR_CONVOLUTION_PRE_PADDINGS, + CUDNN_TYPE_INT64, + spatial_dim_count, + attributes.get_pre_padding().data())); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(wgrad_descriptor.get_raw_desc(), + CUDNN_ATTR_CONVOLUTION_POST_PADDINGS, + CUDNN_TYPE_INT64, + spatial_dim_count, + attributes.get_post_padding().data())); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(wgrad_descriptor.get_raw_desc(), + CUDNN_ATTR_CONVOLUTION_DILATIONS, + CUDNN_TYPE_INT64, + spatial_dim_count, + attributes.get_dilation().data())); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(wgrad_descriptor.get_raw_desc(), + CUDNN_ATTR_CONVOLUTION_FILTER_STRIDES, + CUDNN_TYPE_INT64, + spatial_dim_count, + attributes.get_stride().data())); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(wgrad_descriptor.get_raw_desc())); + CUDNN_FE_LOG_LABEL_ENDL(wgrad_descriptor); + + // Create operation by directly calling cuDNN backend API + Operation_v8 wgrad_operation; + + _CUDNN_CHECK_CUDNN_ERROR(wgrad_operation.initialize_managed_backend_pointer( + CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_FILTER_DESCRIPTOR)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Conv_wgrad_attributes::input_names::X); + auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(wgrad_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_FILTER_X, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &x_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(DY, Conv_wgrad_attributes::input_names::DY); + auto dy_desc = tensors.at(DY->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(wgrad_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_FILTER_DY, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dy_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DW, Conv_wgrad_attributes::output_names::DW); + auto dw_desc = tensors.at(DW->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(wgrad_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_FILTER_DW, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dw_desc)); + + auto conv_desc_ptr = wgrad_descriptor.get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(wgrad_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_FILTER_CONV_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &conv_desc_ptr)); + + float alpha = 1.0f; + float beta = 0.0f; + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(wgrad_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_FILTER_ALPHA, + CUDNN_TYPE_FLOAT, + 1, + &alpha)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(wgrad_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_FILTER_BETA, + CUDNN_TYPE_FLOAT, + 1, + &beta)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(wgrad_operation.get_raw_desc())); + + operations.push_back(std::make_shared<Operation_v8>(std::move(wgrad_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "CONV_WGRAD"})"_json); + } +#endif +}; + +} // namespace cudnn_frontend::graph
\ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/dbn.h b/deps/cudnn-frontend/include/cudnn_frontend/node/dbn.h new file mode 100644 index 0000000..9f2a23e --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/dbn.h @@ -0,0 +1,206 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend { + +namespace graph { + +class DBNNode : public NodeCRTP<DBNNode> { + public: + Batchnorm_backward_attributes attributes; + + DBNNode(Batchnorm_backward_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::DBN; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for DBN node " << attributes.name); + + attributes.fill_from_context(context); + + // TODO: Only inferencing from X works today. + auto X = attributes.inputs[Batchnorm_backward_attributes::input_names::X]; + auto const x_tensor_dim = X->get_dim(); + + auto DX = attributes.outputs[Batchnorm_backward_attributes::output_names::DX]; + auto dx_tensor_dim = DX->get_dim(); + // Only infer dims and strides if user did not set them + if (dx_tensor_dim.empty()) { + dx_tensor_dim.resize(x_tensor_dim.size()); + DX->set_dim(x_tensor_dim); + } + if (DX->get_stride().empty()) { + auto const& DX_dim = DX->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(DX_dim.size()); + DX->set_stride(detail::generate_stride(DX_dim, stride_order)); + } + + // Set channel length tensors + auto infer_per_channel_tensors = [&x_tensor_dim](std::shared_ptr<Tensor_attributes>& T) { + auto tensor_dim = T->get_dim(); + // Only infer dims and strides if user did not set them + if (tensor_dim.empty()) { + tensor_dim.resize(x_tensor_dim.size(), 1); + tensor_dim[1] = x_tensor_dim[1]; + T->set_dim(tensor_dim); + } + if (T->get_stride().empty()) { + auto const& T_dim = T->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(T_dim.size()); + T->set_stride(detail::generate_stride(T_dim, stride_order)); + } + }; + infer_per_channel_tensors(attributes.outputs[Batchnorm_backward_attributes::output_names::DSCALE]); + infer_per_channel_tensors(attributes.outputs[Batchnorm_backward_attributes::output_names::DBIAS]); + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations, + std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: " << "Building DBNNode operations " << attributes.name << " "); + + // Create operation by directly calling cuDNN backend API + Operation_v8 dbn_operation; + + _CUDNN_CHECK_CUDNN_ERROR( + dbn_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_NORM_BACKWARD_DESCRIPTOR)); + + // Set norm mode to BATCH_NORM + cudnnBackendNormMode_t cudnn_norm_mode; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormMode_t::BATCH_NORM, cudnn_norm_mode)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dbn_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_MODE, + CUDNN_TYPE_NORM_MODE, + 1, + &cudnn_norm_mode)); + + // Set input tensor X + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Batchnorm_backward_attributes::input_names::X); + auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dbn_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &x_desc)); + + // Set DY tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(DY, Batchnorm_backward_attributes::input_names::DY); + auto dy_desc = tensors.at(DY->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dbn_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_DYDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dy_desc)); + + // Set scale tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SCALE, Batchnorm_backward_attributes::input_names::SCALE); + auto scale_desc = tensors.at(SCALE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dbn_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_SCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &scale_desc)); + + // Set mean and inv_variance tensors + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(MEAN, Batchnorm_backward_attributes::input_names::MEAN); + auto mean_desc = tensors.at(MEAN->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dbn_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_MEAN_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &mean_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(INV_VARIANCE, + Batchnorm_backward_attributes::input_names::INV_VARIANCE); + auto inv_var_desc = tensors.at(INV_VARIANCE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dbn_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_INV_VARIANCE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &inv_var_desc)); + + // Set DSCALE and DBIAS output tensors + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DSCALE, Batchnorm_backward_attributes::output_names::DSCALE); + auto dscale_desc = tensors.at(DSCALE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dbn_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_DSCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dscale_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DBIAS, Batchnorm_backward_attributes::output_names::DBIAS); + auto dbias_desc = tensors.at(DBIAS->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dbn_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_DBIAS_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dbias_desc)); + + // Set DX output tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DX, Batchnorm_backward_attributes::output_names::DX); + auto dx_desc = tensors.at(DX->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dbn_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_DXDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dx_desc)); + + // Set peer stat tensors if any + if (!attributes.peer_stats.empty()) { + std::vector<cudnnBackendDescriptor_t> peer_stat_descs; + for (auto const& peer_stat : attributes.peer_stats) { + peer_stat_descs.push_back(tensors.at(peer_stat->get_uid())->get_raw_desc()); + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dbn_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_PEER_STAT_DESCS, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + peer_stat_descs.size(), + peer_stat_descs.data())); + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(dbn_operation.get_raw_desc())); + + operations.push_back(std::make_shared<Operation_v8>(std::move(dbn_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "DBN"})"_json); + } +#endif +}; + +} // namespace graph + +} // namespace cudnn_frontend
\ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/dbn_weight.h b/deps/cudnn-frontend/include/cudnn_frontend/node/dbn_weight.h new file mode 100644 index 0000000..bfad0df --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/dbn_weight.h @@ -0,0 +1,215 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend { + +namespace graph { + +class DBNWeightNode : public NodeCRTP<DBNWeightNode> { + public: + DBN_weight_attributes attributes; + + DBNWeightNode(DBN_weight_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::DBN_WEIGHT; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for batchnorm finalize node " << attributes.name); + + attributes.fill_from_context(context); + + // TODO: Only inferencing from DY works today. + auto DY = attributes.inputs[DBN_weight_attributes::input_names::DY]; + auto const dy_tensor_dim = DY->get_dim(); + + auto X = attributes.inputs[DBN_weight_attributes::input_names::X]; + auto x_tensor_dim = X->get_dim(); + // Only infer dims and strides if user did not set them + if (x_tensor_dim.empty()) { + x_tensor_dim.resize(dy_tensor_dim.size()); + X->set_dim(dy_tensor_dim); + } + if (X->get_stride().empty()) { + auto const& X_dim = X->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(X_dim.size()); + X->set_stride(detail::generate_stride(X_dim, stride_order)); + } + + // Set channel length tensors + auto infer_per_channel_tensors = [&dy_tensor_dim](std::shared_ptr<Tensor_attributes> const& T) { + auto tensor_dim = T->get_dim(); + // Only infer dims and strides if user did not set them + if (T->get_dim().empty()) { + tensor_dim.resize(dy_tensor_dim.size(), 1); + tensor_dim[1] = dy_tensor_dim[1]; + T->set_dim(tensor_dim); + } + if (T->get_stride().empty()) { + auto const& T_dim = T->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(T_dim.size()); + T->set_stride(detail::generate_stride(T_dim, stride_order)); + } + }; + infer_per_channel_tensors(attributes.outputs[DBN_weight_attributes::output_names::DBIAS]); + infer_per_channel_tensors(attributes.outputs[DBN_weight_attributes::output_names::DSCALE]); + infer_per_channel_tensors(attributes.outputs[DBN_weight_attributes::output_names::EQ_BIAS]); + infer_per_channel_tensors(attributes.outputs[DBN_weight_attributes::output_names::EQ_SCALE_DY]); + infer_per_channel_tensors(attributes.outputs[DBN_weight_attributes::output_names::EQ_SCALE_X]); + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations, + std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: " << "Building DBNWeightNode operations " << attributes.name << " "); + + // Create operation by directly calling cuDNN backend API + Operation_v8 bn_bwd_weight_operation; + + _CUDNN_CHECK_CUDNN_ERROR(bn_bwd_weight_operation.initialize_managed_backend_pointer( + CUDNN_BACKEND_OPERATION_BN_BWD_WEIGHTS_DESCRIPTOR)); + + // Set compute type (math precision) + cudnnDataType_t compute_type = CUDNN_DATA_FLOAT; + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_bwd_weight_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_MATH_PREC, + CUDNN_TYPE_DATA_TYPE, + 1, + &compute_type)); + + // Set input tensor X + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, DBN_weight_attributes::input_names::X); + auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_bwd_weight_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_X_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &x_desc)); + + // Set DY tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(DY, DBN_weight_attributes::input_names::DY); + auto dy_desc = tensors.at(DY->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_bwd_weight_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_DY_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dy_desc)); + + // Set mean tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(MEAN, DBN_weight_attributes::input_names::MEAN); + auto mean_desc = tensors.at(MEAN->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_bwd_weight_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_MEAN_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &mean_desc)); + + // Set inv_variance tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(INV_VARIANCE, DBN_weight_attributes::input_names::INV_VARIANCE); + auto inv_var_desc = tensors.at(INV_VARIANCE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_bwd_weight_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_INVSTD_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &inv_var_desc)); + + // Set scale tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SCALE, DBN_weight_attributes::input_names::SCALE); + auto scale_desc = tensors.at(SCALE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_bwd_weight_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_BN_SCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &scale_desc)); + + // Set DSCALE output tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DSCALE, DBN_weight_attributes::output_names::DSCALE); + auto dscale_desc = tensors.at(DSCALE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_bwd_weight_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_DBN_SCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dscale_desc)); + + // Set DBIAS output tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DBIAS, DBN_weight_attributes::output_names::DBIAS); + auto dbias_desc = tensors.at(DBIAS->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_bwd_weight_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_DBN_BIAS_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dbias_desc)); + + // Set EQ_SCALE_DY output tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(EQ_SCALE_DY, DBN_weight_attributes::output_names::EQ_SCALE_DY); + auto eq_scale_dy_desc = tensors.at(EQ_SCALE_DY->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_bwd_weight_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_EQ_DY_SCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &eq_scale_dy_desc)); + + // Set EQ_SCALE_X output tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(EQ_SCALE_X, DBN_weight_attributes::output_names::EQ_SCALE_X); + auto eq_scale_x_desc = tensors.at(EQ_SCALE_X->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_bwd_weight_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_EQ_X_SCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &eq_scale_x_desc)); + + // Set EQ_BIAS output tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(EQ_BIAS, DBN_weight_attributes::output_names::EQ_BIAS); + auto eq_bias_desc = tensors.at(EQ_BIAS->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_bwd_weight_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_EQ_BIAS, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &eq_bias_desc)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(bn_bwd_weight_operation.get_raw_desc())); + + operations.push_back(std::make_shared<Operation_v8>(std::move(bn_bwd_weight_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "DBN_WEIGHT"})"_json); + } +#endif +}; + +} // namespace graph + +} // namespace cudnn_frontend
\ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/diagonal_band_mask.h b/deps/cudnn-frontend/include/cudnn_frontend/node/diagonal_band_mask.h new file mode 100644 index 0000000..243d1e3 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/diagonal_band_mask.h @@ -0,0 +1,474 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +#include "pointwise.h" +#include "reduction.h" + +namespace cudnn_frontend::graph { + +// Base class for nodes that represent a diagonal band mask operation +// (which is either a left-bound or a right-bound mask, but not both at the same time). +template <typename DerivedT> +class DiagonalBandMaskNodeBase : public NodeCRTP<DerivedT> { + protected: + using input_names = DiagonalBandMask_attributes::input_names; + using output_names = DiagonalBandMask_attributes::output_names; + + public: + DiagonalBandMask_attributes attributes; + + DiagonalBandMaskNodeBase(DiagonalBandMask_attributes&& attributes_, detail::Context const& context) + : NodeCRTP<DerivedT>(context), attributes(std::move(attributes_)) {} + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating DiagonalBandMaskNode " << attributes.name); + + RETURN_CUDNN_FRONTEND_ERROR_IF(has_left_bound() && has_shift_right_bound(), + error_code_t::INVALID_VALUE, + "DiagonalBandMaskNode cannot have both left_bound and shift_right_bound"); + RETURN_CUDNN_FRONTEND_ERROR_IF( + (has_seq_len_q() || has_seq_len_kv()) && (has_cu_seq_len_q() || has_cu_seq_len_kv()), + error_code_t::INVALID_VALUE, + "SEQ_LEN_Q / SEQ_LEN_KV and CU_SEQ_LEN_Q / CU_SEQ_LEN_KV are mutually exclusive"); + + return pre_validate_node_extra(); + } + + virtual error_t + pre_validate_node_extra() const { + return {error_code_t::OK, ""}; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for DiagonalBandMask node " << attributes.name); + + attributes.fill_from_context(this->context); + + auto input_tensor = attributes.inputs.at(DiagonalBandMask_attributes::input_names::X); + auto output_tensor = attributes.outputs.at(DiagonalBandMask_attributes::output_names::Y); + + // Only infer dims and strides if user did not set them + if (output_tensor->get_dim().empty()) { + output_tensor->set_dim(input_tensor->get_dim()); + } + + if (output_tensor->get_stride().empty()) { + output_tensor->set_stride(input_tensor->get_stride()); + } + + return {error_code_t::OK, ""}; + } + + protected: + bool + has_seq_len_q() const { + auto seq_len_Q_it = attributes.inputs.find(DiagonalBandMask_attributes::input_names::SEQ_LEN_Q); + return ((seq_len_Q_it) != attributes.inputs.end() && seq_len_Q_it->second != nullptr); + } + + bool + has_seq_len_kv() const { + auto seq_len_KV_it = attributes.inputs.find(DiagonalBandMask_attributes::input_names::SEQ_LEN_KV); + return ((seq_len_KV_it) != attributes.inputs.end() && seq_len_KV_it->second != nullptr); + } + + bool + has_cu_seq_len_q() const { + auto cu_seq_len_Q_it = attributes.inputs.find(DiagonalBandMask_attributes::input_names::CU_SEQ_LEN_Q); + return ((cu_seq_len_Q_it) != attributes.inputs.end() && cu_seq_len_Q_it->second != nullptr); + } + + bool + has_cu_seq_len_kv() const { + auto cu_seq_len_KV_it = attributes.inputs.find(DiagonalBandMask_attributes::input_names::CU_SEQ_LEN_KV); + return ((cu_seq_len_KV_it) != attributes.inputs.end() && cu_seq_len_KV_it->second != nullptr); + } + + bool + has_left_bound() const { + auto left_bound_it = attributes.inputs.find(DiagonalBandMask_attributes::input_names::LeftBound); + return ((left_bound_it) != attributes.inputs.end() && left_bound_it->second != nullptr); + } + + bool + has_right_bound() const { + return !has_left_bound(); + } + + bool + has_shift_right_bound() const { + auto shift_right_bound_it = attributes.inputs.find(DiagonalBandMask_attributes::input_names::ShiftRightBound); + return ((shift_right_bound_it) != attributes.inputs.end() && shift_right_bound_it->second != nullptr); + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + } +#endif +}; + +// Fallback implementation of diagonal band mask node that represents the mask as a series of pointwise operations. +// This is used for cuDNN versions before v9.21.0. +// Note: the individual masks can be constructed in different ways as well that yield functionally +// correct results. However, for performance reasons in the cuDNN backend they are organized as they are. Be +// cautious of performance when editing. +class CompositeDiagonalBandMaskNode : public DiagonalBandMaskNodeBase<CompositeDiagonalBandMaskNode> { + public: + CompositeDiagonalBandMaskNode(DiagonalBandMask_attributes&& attributes_, detail::Context const& context) + : DiagonalBandMaskNodeBase(std::move(attributes_), context) {} + + Type + getType() override final { + return Type::COMPOSITE; + } + + error_t + pre_validate_node_extra() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating CompositeDiagonalBandMaskNode " << attributes.name); + + RETURN_CUDNN_FRONTEND_ERROR_IF(has_cu_seq_len_q() || has_cu_seq_len_kv(), + error_code_t::GRAPH_NOT_SUPPORTED, + "CompositeDiagonalBandMaskNode does not support CU_SEQ_LEN_Q or CU_SEQ_LEN_KV"); + + return {error_code_t::OK, ""}; + } + + error_t + expand_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for CompositeDiagonalBandMaskNode " + << attributes.name); + + attributes.fill_from_context(context); + + auto attention_score = attributes.inputs[DiagonalBandMask_attributes::input_names::X]; + auto negative_inf_value = attributes.inputs[DiagonalBandMask_attributes::input_names::B]; + + auto row_index = pointwise( + attention_score, + Pointwise_attributes().set_name("gen_row_index").set_mode(PointwiseMode_t::GEN_INDEX).set_axis(2)); + row_index->set_data_type(DataType_t::INT32); + auto col_index = pointwise( + attention_score, + Pointwise_attributes().set_name("gen_col_index").set_mode(PointwiseMode_t::GEN_INDEX).set_axis(3)); + col_index->set_data_type(DataType_t::INT32); + + std::shared_ptr<Tensor_attributes> swa_comparison_output; + + if (has_right_bound()) { + std::string row_index_name = "row"; + + if (has_seq_len_kv()) { + auto seq_len_kv = attributes.inputs[DiagonalBandMask_attributes::input_names::SEQ_LEN_KV]; + row_index = pointwise(row_index, + seq_len_kv, + Pointwise_attributes() + .set_name(row_index_name += "+skv") + .set_mode(PointwiseMode_t::ADD) + .set_compute_data_type(DataType_t::INT32) + .set_axis(3)); + row_index->set_data_type(DataType_t::INT32); + } + + if (has_seq_len_q()) { + auto seq_len_q = attributes.inputs[DiagonalBandMask_attributes::input_names::SEQ_LEN_Q]; + row_index = pointwise(row_index, + seq_len_q, + Pointwise_attributes() + .set_name(row_index_name += "-sq") + .set_mode(PointwiseMode_t::SUB) + .set_compute_data_type(DataType_t::INT32) + .set_axis(3)); + row_index->set_data_type(DataType_t::INT32); + } + + if (has_shift_right_bound()) { + auto shift_right_bound = attributes.inputs[DiagonalBandMask_attributes::input_names::ShiftRightBound]; + // Use shift_right_bound's data type + row_index = pointwise(row_index, + shift_right_bound, + Pointwise_attributes() + .set_name(row_index_name += "+window") + .set_mode(PointwiseMode_t::ADD) + .set_compute_data_type(shift_right_bound->get_data_type()) + .set_axis(3)); + row_index->set_data_type(shift_right_bound->get_data_type()); + } + + swa_comparison_output = pointwise(row_index, + col_index, + Pointwise_attributes() + .set_name(row_index_name += ">col") + .set_mode(attributes.comparison_mode) + .set_compute_data_type(DataType_t::BOOLEAN)); + swa_comparison_output->set_data_type(DataType_t::BOOLEAN); + } + + // Set the left bound + else if (has_left_bound()) { + auto left_bound = attributes.inputs[DiagonalBandMask_attributes::input_names::LeftBound]; + std::string col_index_name = "col"; + + // Use left_bound's data type + col_index = pointwise(col_index, + left_bound, + Pointwise_attributes() + .set_name(col_index_name += "+window") + .set_mode(PointwiseMode_t::ADD) + .set_compute_data_type(left_bound->get_data_type()) + .set_axis(3)); + col_index->set_data_type(left_bound->get_data_type()); + + if (has_seq_len_kv()) { + auto seq_len_kv = attributes.inputs[DiagonalBandMask_attributes::input_names::SEQ_LEN_KV]; + col_index = pointwise(col_index, + seq_len_kv, + Pointwise_attributes() + .set_name(col_index_name += "-skv") + .set_mode(PointwiseMode_t::SUB) + .set_compute_data_type(DataType_t::INT32) + .set_axis(3)); + col_index->set_data_type(DataType_t::INT32); + } + + if (has_seq_len_q()) { + auto seq_len_q = attributes.inputs[DiagonalBandMask_attributes::input_names::SEQ_LEN_Q]; + col_index = pointwise(col_index, + seq_len_q, + Pointwise_attributes() + .set_name(col_index_name += "+sq") + .set_mode(PointwiseMode_t::ADD) + .set_compute_data_type(DataType_t::INT32) + .set_axis(3)); + col_index->set_data_type(DataType_t::INT32); + } + + swa_comparison_output = pointwise(col_index, + row_index, + Pointwise_attributes() + .set_name(col_index_name += ">row") + .set_mode(attributes.comparison_mode) + .set_compute_data_type(DataType_t::BOOLEAN)); + swa_comparison_output->set_data_type(DataType_t::BOOLEAN); + } else { + // This should not be reached (since one of has_left_bound() or has_right_bound() is always true by + // construction), just here for completeness. + return {error_code_t::INVALID_VALUE, + "CompositeDiagonalBandMaskNode must have a left bound or a right bound"}; + } + + // Special non-functional-style call. Needed because output already created and provided to user. + pointwise(attention_score, + negative_inf_value, + swa_comparison_output, + Pointwise_attributes().set_name("binary_select").set_mode(PointwiseMode_t::BINARY_SELECT), + attributes.outputs[DiagonalBandMask_attributes::output_names::Y]); + + return {error_code_t::OK, ""}; + } +}; + +// Newer implementation of diagonal band mask node that represents the mask as a single operation. +// This is used for cuDNN versions v9.21.0 and later. +class UnifiedDiagonalBandMaskNode : public DiagonalBandMaskNodeBase<UnifiedDiagonalBandMaskNode> { + public: + UnifiedDiagonalBandMaskNode(DiagonalBandMask_attributes&& attributes_, detail::Context const& context) + : DiagonalBandMaskNodeBase(std::move(attributes_), context) {} + + Type + getType() override final { + return Type::DIAGONAL_BAND_MASK; + } + + error_t + create_cudnn_operations( + std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations, + std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final { + getLogger() << "[cudnn_frontend] INFO: " << "Building UnifiedDiagonalBandMaskNode operations " + << attributes.name << std::endl; + auto cudnn_ver_error = + error_t{error_code_t::GRAPH_NOT_SUPPORTED, "UnifiedDiagonalBandMaskNode requires cuDNN v9.21.0"}; + +#if (CUDNN_VERSION >= 92100) + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(92100, cudnn_ver_error); + CUDNN_FRONTEND_UNUSED(operations); + auto diagonal_band_mask_operation = make_shared_backend_pointer( + (cudnnBackendDescriptorType_t)CUDNN_BACKEND_OPERATION_DIAGONAL_BAND_MASK_DESCRIPTOR); + + auto X = attributes.inputs.find(DiagonalBandMask_attributes::input_names::X)->second; + auto backend_x = tensors[X->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(diagonal_band_mask_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_x)); + + auto B = attributes.inputs.find(DiagonalBandMask_attributes::input_names::B)->second; + auto backend_b = tensors[B->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(diagonal_band_mask_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_BDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_b)); + + if (has_seq_len_q()) { + auto seq_len_Q = attributes.inputs.find(DiagonalBandMask_attributes::input_names::SEQ_LEN_Q)->second; + auto backend_seq_len_Q = tensors[seq_len_Q->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(diagonal_band_mask_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_SEQ_LEN_QDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_seq_len_Q)); + } + + if (has_seq_len_kv()) { + auto seq_len_KV = attributes.inputs.find(DiagonalBandMask_attributes::input_names::SEQ_LEN_KV)->second; + auto backend_seq_len_KV = tensors[seq_len_KV->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(diagonal_band_mask_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_SEQ_LEN_KVDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_seq_len_KV)); + } + + if (has_cu_seq_len_q() || has_cu_seq_len_kv()) { + auto cu_seq_len_cudnn_ver_error = + error_t{error_code_t::GRAPH_NOT_SUPPORTED, + "Cumulative sequence length tensors in unified DiagonalBandMask node require cuDNN 9.24.0"}; +#if (CUDNN_VERSION >= 92400) + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(92400, cu_seq_len_cudnn_ver_error); + + if (has_cu_seq_len_q()) { + auto cu_seq_len_Q = + attributes.inputs.find(DiagonalBandMask_attributes::input_names::CU_SEQ_LEN_Q)->second; + auto backend_cu_seq_len_Q = tensors[cu_seq_len_Q->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(diagonal_band_mask_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_CU_SEQ_LEN_QDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_cu_seq_len_Q)); + } + + if (has_cu_seq_len_kv()) { + auto cu_seq_len_KV = + attributes.inputs.find(DiagonalBandMask_attributes::input_names::CU_SEQ_LEN_KV)->second; + auto backend_cu_seq_len_KV = tensors[cu_seq_len_KV->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR( + detail::set_attribute(diagonal_band_mask_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_CU_SEQ_LEN_KVDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_cu_seq_len_KV)); + } +#else + return cu_seq_len_cudnn_ver_error; +#endif + } + + if (has_left_bound()) { + auto left_bound = attributes.inputs.find(DiagonalBandMask_attributes::input_names::LeftBound)->second; + auto backend_left_bound = tensors[left_bound->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(diagonal_band_mask_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_LEFT_BOUND_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_left_bound)); + } + + if (has_shift_right_bound()) { + auto shift_right_bound = + attributes.inputs.find(DiagonalBandMask_attributes::input_names::ShiftRightBound)->second; + auto backend_shift_right_bound = + tensors[shift_right_bound->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR( + detail::set_attribute(diagonal_band_mask_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_SHIFT_RIGHT_BOUND_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_shift_right_bound)); + } + + auto Y = attributes.outputs.find(DiagonalBandMask_attributes::output_names::Y)->second; + auto backend_y = tensors[Y->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(diagonal_band_mask_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_YDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_y)); + + cudnnPointwiseMode_t cudnn_pointwise_mode; + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.comparison_mode, cudnn_pointwise_mode)); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(diagonal_band_mask_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_COMPARISON_MODE, + CUDNN_TYPE_POINTWISE_MODE, + 1, + &cudnn_pointwise_mode)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(diagonal_band_mask_operation->get_backend_descriptor())); + + raw_operations.push_back(diagonal_band_mask_operation); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; +#else + CUDNN_FRONTEND_UNUSED(uids_involved_in_operations); + CUDNN_FRONTEND_UNUSED(operations); + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FRONTEND_UNUSED(tensors); + return cudnn_ver_error; +#endif // CUDNN_VERSION >= 92100 + } +}; + +// Factory function to create a diagonal band mask node. +// This function will create the appropriate node based on the cuDNN version. +inline std::shared_ptr<Tensor_attributes> +INode::diagonal_band_mask(std::shared_ptr<Tensor_attributes> x, + std::shared_ptr<Tensor_attributes> b, + std::shared_ptr<Tensor_attributes> seq_len_q, + std::shared_ptr<Tensor_attributes> seq_len_kv, + std::shared_ptr<Tensor_attributes> cu_seq_len_q, + std::shared_ptr<Tensor_attributes> cu_seq_len_kv, + std::shared_ptr<Tensor_attributes> left_bound, + std::shared_ptr<Tensor_attributes> shift_right_bound, + DiagonalBandMask_attributes attributes) { + attributes.inputs[DiagonalBandMask_attributes::input_names::X] = x; + attributes.inputs[DiagonalBandMask_attributes::input_names::B] = b; + if (seq_len_q) { + attributes.inputs[DiagonalBandMask_attributes::input_names::SEQ_LEN_Q] = seq_len_q; + } + if (seq_len_kv) { + attributes.inputs[DiagonalBandMask_attributes::input_names::SEQ_LEN_KV] = seq_len_kv; + } + if (cu_seq_len_q) { + attributes.inputs[DiagonalBandMask_attributes::input_names::CU_SEQ_LEN_Q] = cu_seq_len_q; + } + if (cu_seq_len_kv) { + attributes.inputs[DiagonalBandMask_attributes::input_names::CU_SEQ_LEN_KV] = cu_seq_len_kv; + } + if (left_bound) { + attributes.inputs[DiagonalBandMask_attributes::input_names::LeftBound] = left_bound; + } + if (shift_right_bound) { + attributes.inputs[DiagonalBandMask_attributes::input_names::ShiftRightBound] = shift_right_bound; + } + auto y = attributes.outputs[DiagonalBandMask_attributes::output_names::Y] = output_tensor(attributes.name + "::Y"); + + // Newer versions of cuDNN can represent the diagonal band mask as a single operation. + if (std::min(detail::get_compiled_version(), detail::get_backend_version()) >= 92100) { + sub_nodes.emplace_back(std::make_unique<UnifiedDiagonalBandMaskNode>(std::move(attributes), context)); + } else { + sub_nodes.emplace_back(std::make_unique<CompositeDiagonalBandMaskNode>(std::move(attributes), context)); + } + return y; +} + +} // namespace cudnn_frontend::graph
\ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/dln.h b/deps/cudnn-frontend/include/cudnn_frontend/node/dln.h new file mode 100644 index 0000000..ad7ad1f --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/dln.h @@ -0,0 +1,226 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend { + +namespace graph { + +class DLNNode : public NodeCRTP<DLNNode> { + public: + Layernorm_backward_attributes attributes; + + DLNNode(Layernorm_backward_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::DLN; + } + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for DLN node " << attributes.name); + + if (detail::get_backend_version() < 8906) { + attributes.inputs[Layernorm_backward_attributes::input_names::EPSILON] = + std::make_shared<Tensor_attributes>(0.0f); + } + + attributes.fill_from_context(context); + + // TODO: Only inferencing from X works today. + auto X = attributes.inputs[Layernorm_backward_attributes::input_names::X]; + auto const x_tensor_dim = X->get_dim(); + + auto DY = attributes.inputs[Layernorm_backward_attributes::input_names::DY]; + auto dy_tensor_dim = DY->get_dim(); + + // Only infer dims and strides if user did not set them + if (dy_tensor_dim.empty()) { + dy_tensor_dim.resize(x_tensor_dim.size()); + DY->set_dim(x_tensor_dim); + } + if (DY->get_stride().empty()) { + auto const& DY_dim = DY->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(DY_dim.size()); + DY->set_stride(detail::generate_stride(DY_dim, stride_order)); + } + + auto DX = attributes.outputs[Layernorm_backward_attributes::output_names::DX]; + auto dx_tensor_dim = DX->get_dim(); + // Only infer dims and strides if user did not set them + if (dx_tensor_dim.empty()) { + dx_tensor_dim.resize(x_tensor_dim.size()); + DX->set_dim(x_tensor_dim); + } + if (DX->get_stride().empty()) { + auto const& DX_dim = DX->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(DX_dim.size()); + DX->set_stride(detail::generate_stride(DX_dim, stride_order)); + } + + auto scale_bias_dim = X->get_dim(); + scale_bias_dim[0] = 1; + + // Set channel length tensors + auto infer_scale_bias_tensors = [&scale_bias_dim](std::shared_ptr<Tensor_attributes>& T) { + auto tensor_dim = T->get_dim(); + // Only infer dims and strides if user did not set them + if (tensor_dim.empty()) { + T->set_dim(scale_bias_dim); + } + if (T->get_stride().empty()) { + auto const& T_dim = T->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(T_dim.size()); + T->set_stride(detail::generate_stride(T_dim, stride_order)); + } + }; + + infer_scale_bias_tensors(attributes.outputs[Layernorm_backward_attributes::output_names::DSCALE]); + infer_scale_bias_tensors(attributes.outputs[Layernorm_backward_attributes::output_names::DBIAS]); + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations, + std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: " << "Building DLNNode operations " << attributes.name << " "); + + // Create operation by directly calling cuDNN backend API + Operation_v8 dln_operation; + + _CUDNN_CHECK_CUDNN_ERROR( + dln_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_NORM_BACKWARD_DESCRIPTOR)); + + // Set norm mode to LAYER_NORM + cudnnBackendNormMode_t cudnn_norm_mode; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormMode_t::LAYER_NORM, cudnn_norm_mode)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dln_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_MODE, + CUDNN_TYPE_NORM_MODE, + 1, + &cudnn_norm_mode)); + + // Set input tensor X + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Layernorm_backward_attributes::input_names::X); + auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dln_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &x_desc)); + + // Set DY tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(DY, Layernorm_backward_attributes::input_names::DY); + auto dy_desc = tensors.at(DY->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dln_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_DYDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dy_desc)); + + // Set scale tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SCALE, Layernorm_backward_attributes::input_names::SCALE); + auto scale_desc = tensors.at(SCALE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dln_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_SCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &scale_desc)); + + // Set mean and inv_variance tensors + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(MEAN, Layernorm_backward_attributes::input_names::MEAN); + auto mean_desc = tensors.at(MEAN->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dln_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_MEAN_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &mean_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(INV_VARIANCE, + Layernorm_backward_attributes::input_names::INV_VARIANCE); + auto inv_var_desc = tensors.at(INV_VARIANCE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dln_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_INV_VARIANCE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &inv_var_desc)); + + // Set DSCALE and DBIAS output tensors + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DSCALE, Layernorm_backward_attributes::output_names::DSCALE); + auto dscale_desc = tensors.at(DSCALE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dln_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_DSCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dscale_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DBIAS, Layernorm_backward_attributes::output_names::DBIAS); + auto dbias_desc = tensors.at(DBIAS->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dln_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_DBIAS_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dbias_desc)); + + // Set DX output tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DX, Layernorm_backward_attributes::output_names::DX); + auto dx_desc = tensors.at(DX->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dln_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_DXDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dx_desc)); + + // Set epsilon tensor for older backend versions + if (detail::get_backend_version() < 8906) { + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(EPSILON, Layernorm_backward_attributes::input_names::EPSILON); + auto epsilon_desc = tensors.at(EPSILON->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dln_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_EPSILON_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &epsilon_desc)); + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(dln_operation.get_raw_desc())); + + operations.push_back(std::make_shared<Operation_v8>(std::move(dln_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "LAYER_NORM_BPROP"})"_json); + } +#endif +}; + +} // namespace graph + +} // namespace cudnn_frontend diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/genstats.h b/deps/cudnn-frontend/include/cudnn_frontend/node/genstats.h new file mode 100644 index 0000000..8f91897 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/genstats.h @@ -0,0 +1,147 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend { + +namespace graph { + +class GenstatsNode : public NodeCRTP<GenstatsNode> { + public: + Genstats_attributes attributes; + + GenstatsNode(Genstats_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::GENSTATS; + } + + error_t + infer_properties_node() override final { + attributes.fill_from_context(context); + + // Only inferrencing from X works today. + auto X = attributes.inputs[Genstats_attributes::input_names::X]; + auto SUM = attributes.outputs[Genstats_attributes::output_names::SUM]; + auto SQ_SUM = attributes.outputs[Genstats_attributes::output_names::SQ_SUM]; + + auto const x_tensor_dim = X->get_dim(); + auto sum_tensor_dim = SUM->get_dim(); + auto sq_sum_tensor_dim = SQ_SUM->get_dim(); + + // Only infer dims and strides if user did not set them + if (sum_tensor_dim.empty()) { + sum_tensor_dim.resize(x_tensor_dim.size(), 1); + sum_tensor_dim[1] = x_tensor_dim[1]; + SUM->set_dim(sum_tensor_dim); + } + if (SUM->get_stride().empty()) { + auto const& SUM_dim = SUM->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(SUM_dim.size()); + SUM->set_stride(detail::generate_stride(SUM_dim, stride_order)); + } + + // Only infer dims and strides if user did not set them + if (sq_sum_tensor_dim.empty()) { + sq_sum_tensor_dim.resize(x_tensor_dim.size(), 1); + sq_sum_tensor_dim[1] = x_tensor_dim[1]; + SQ_SUM->set_dim(sq_sum_tensor_dim); + } + if (SQ_SUM->get_stride().empty()) { + auto const& SQ_SUM_dim = SQ_SUM->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(SQ_SUM_dim.size()); + SQ_SUM->set_stride(detail::generate_stride(SQ_SUM_dim, stride_order)); + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations, + std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: " << "Building GenstatsNode operations " << attributes.name << " "); + + // Create operation by directly calling cuDNN backend API + Operation_v8 genstats_operation; + + _CUDNN_CHECK_CUDNN_ERROR( + genstats_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_GEN_STATS_DESCRIPTOR)); + + // Set input tensor X + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Genstats_attributes::input_names::X); + auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(genstats_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_GENSTATS_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &x_desc)); + + // Set gen stats mode + cudnnGenStatsMode_t genstats_mode = CUDNN_GENSTATS_SUM_SQSUM; + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(genstats_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_GENSTATS_MODE, + CUDNN_TYPE_GENSTATS_MODE, + 1, + &genstats_mode)); + + // Set math precision based on X tensor data type + cudnnDataType_t math_prec = static_cast<cudnnDataType_t>(tensors.at(X->second->get_uid())->getDataType()); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(genstats_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_GENSTATS_MATH_PREC, + CUDNN_TYPE_DATA_TYPE, + 1, + &math_prec)); + + // Set SUM output tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(SUM, Genstats_attributes::output_names::SUM); + auto sum_desc = tensors.at(SUM->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(genstats_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_GENSTATS_SUMDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &sum_desc)); + + // Set SQ_SUM output tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(SQ_SUM, Genstats_attributes::output_names::SQ_SUM); + auto sq_sum_desc = tensors.at(SQ_SUM->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(genstats_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_GENSTATS_SQSUMDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &sq_sum_desc)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(genstats_operation.get_raw_desc())); + + operations.push_back(std::make_shared<Operation_v8>(std::move(genstats_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "GENSTATS"})"_json); + } +#endif +}; + +} // namespace graph + +} // namespace cudnn_frontend
\ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/instancenorm.h b/deps/cudnn-frontend/include/cudnn_frontend/node/instancenorm.h new file mode 100644 index 0000000..1b71f4a --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/instancenorm.h @@ -0,0 +1,414 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend { + +namespace graph { +class InstanceNormNode : public NodeCRTP<InstanceNormNode> { + public: + Instancenorm_attributes attributes; + + InstanceNormNode(Instancenorm_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::INSTANCENORM; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for instancenorm node " << attributes.name); + + attributes.fill_from_context(context); + + auto X = attributes.inputs[Instancenorm_attributes::input_names::X]; + auto Y = attributes.outputs[Instancenorm_attributes::output_names::Y]; + + // Only infer dims and strides if user did not set them + if (Y->get_dim().empty()) { + Y->set_dim(X->get_dim()); + } + if (Y->get_stride().empty()) { + Y->set_stride(X->get_stride()); + } + + // mean inv_var dim is n,c,1,1 + auto stats_dim = X->get_dim(); + for (size_t i = 2; i < stats_dim.size(); i++) { + stats_dim[i] = 1; + } + + if (attributes.forward_phase == NormFwdPhase_t::TRAINING) { + auto mean = attributes.outputs[Instancenorm_attributes::output_names::MEAN]; + if (mean->get_dim().empty()) { + mean->set_dim(stats_dim); + } + if (mean->get_stride().empty()) { + auto const& mean_dim = mean->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(mean_dim.size()); + mean->set_stride(detail::generate_stride(mean_dim, stride_order)); + } + + auto inv_var = attributes.outputs[Instancenorm_attributes::output_names::INV_VARIANCE]; + if (inv_var->get_dim().empty()) { + inv_var->set_dim(stats_dim); + } + if (inv_var->get_stride().empty()) { + auto const& inv_var_dim = inv_var->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(inv_var_dim.size()); + inv_var->set_stride(detail::generate_stride(inv_var_dim, stride_order)); + } + } + return {error_code_t::OK, ""}; + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating InstanceNormNode " << attributes.name); + + // Norm forward phase should be set + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.forward_phase == NormFwdPhase_t::NOT_SET, + error_code_t::ATTRIBUTE_NOT_SET, + "Forward phase not set of instancenorm node."); + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations, + std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: Building InstanceNormNode operations " << attributes.name << " "); + + // Create operation by directly calling cuDNN backend API + Operation_v8 instancenorm_operation; + + _CUDNN_CHECK_CUDNN_ERROR( + instancenorm_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_NORM_FORWARD_DESCRIPTOR)); + + // Set norm mode to INSTANCE_NORM + cudnnBackendNormMode_t cudnn_norm_mode; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormMode_t::INSTANCE_NORM, cudnn_norm_mode)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(instancenorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_MODE, + CUDNN_TYPE_NORM_MODE, + 1, + &cudnn_norm_mode)); + + // Set forward phase + cudnnBackendNormFwdPhase_t cudnn_norm_fwd_phase; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.forward_phase, cudnn_norm_fwd_phase)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(instancenorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_PHASE, + CUDNN_TYPE_NORM_FWD_PHASE, + 1, + &cudnn_norm_fwd_phase)); + + // Set input tensor X + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Instancenorm_attributes::input_names::X); + auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(instancenorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &x_desc)); + + // Set scale and bias tensors + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SCALE, Instancenorm_attributes::input_names::SCALE); + auto scale_desc = tensors.at(SCALE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(instancenorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_SCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &scale_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(BIAS, Instancenorm_attributes::input_names::BIAS); + auto bias_desc = tensors.at(BIAS->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(instancenorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_BIAS_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &bias_desc)); + + // Set epsilon tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(EPSILON, Instancenorm_attributes::input_names::EPSILON); + auto epsilon_desc = tensors.at(EPSILON->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(instancenorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_EPSILON_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &epsilon_desc)); + + // Set output tensor Y + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(Y, Instancenorm_attributes::output_names::Y); + auto y_desc = tensors.at(Y->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(instancenorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_YDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &y_desc)); + + // Set mean and inv_variance for training phase + if (attributes.forward_phase == NormFwdPhase_t::TRAINING) { + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(MEAN, Instancenorm_attributes::output_names::MEAN); + auto mean_desc = tensors.at(MEAN->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(instancenorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_MEAN_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &mean_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(INV_VARIANCE, + Instancenorm_attributes::output_names::INV_VARIANCE); + auto inv_var_desc = tensors.at(INV_VARIANCE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(instancenorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_INV_VARIANCE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &inv_var_desc)); + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(instancenorm_operation.get_raw_desc())); + + operations.push_back(std::make_shared<Operation_v8>(std::move(instancenorm_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "INSTANCE_NORM"})"_json); + } +#endif +}; + +class DINNode : public NodeCRTP<DINNode> { + public: + Instancenorm_backward_attributes attributes; + + DINNode(Instancenorm_backward_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::DIN; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for DIN node " << attributes.name); + + attributes.fill_from_context(context); + + // TODO: Only inferencing from X works today. + auto X = attributes.inputs[Instancenorm_backward_attributes::input_names::X]; + auto const x_tensor_dim = X->get_dim(); + + auto DY = attributes.inputs[Instancenorm_backward_attributes::input_names::DY]; + auto dy_tensor_dim = DY->get_dim(); + + // Only infer dims and strides if user did not set them + if (dy_tensor_dim.empty()) { + dy_tensor_dim.resize(x_tensor_dim.size()); + DY->set_dim(x_tensor_dim); + } + if (DY->get_stride().empty()) { + auto const& DY_dim = DY->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(DY_dim.size()); + DY->set_stride(detail::generate_stride(DY_dim, stride_order)); + } + + auto DX = attributes.outputs[Instancenorm_backward_attributes::output_names::DX]; + auto dx_tensor_dim = DX->get_dim(); + // Only infer dims and strides if user did not set them + if (dx_tensor_dim.empty()) { + dx_tensor_dim.resize(x_tensor_dim.size()); + DX->set_dim(x_tensor_dim); + } + if (DX->get_stride().empty()) { + auto const& DX_dim = DX->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(DX_dim.size()); + DX->set_stride(detail::generate_stride(DX_dim, stride_order)); + } + + // scale_bias dim is 1,c,1,1 + // mean inv_var dim is n,c,1,1 + auto scale_bias_dim = X->get_dim(); + for (size_t i = 0; i < scale_bias_dim.size(); i++) { + if (i != 1) { + scale_bias_dim[i] = 1; + } + } + + // Set channel length tensors + auto infer_scale_bias_tensors = [&scale_bias_dim](std::shared_ptr<Tensor_attributes>& T) { + auto tensor_dim = T->get_dim(); + // Only infer dims and strides if user did not set them + if (tensor_dim.empty()) { + T->set_dim(scale_bias_dim); + } + if (T->get_stride().empty()) { + auto const& T_dim = T->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(T_dim.size()); + T->set_stride(detail::generate_stride(T_dim, stride_order)); + } + }; + + infer_scale_bias_tensors(attributes.outputs[Instancenorm_backward_attributes::output_names::DSCALE]); + infer_scale_bias_tensors(attributes.outputs[Instancenorm_backward_attributes::output_names::DBIAS]); + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations, + std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: Building DINNode operations " << attributes.name << " "); + + // Create operation by directly calling cuDNN backend API + Operation_v8 din_operation; + + _CUDNN_CHECK_CUDNN_ERROR( + din_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_NORM_BACKWARD_DESCRIPTOR)); + + // Set norm mode to INSTANCE_NORM + cudnnBackendNormMode_t cudnn_norm_mode; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormMode_t::INSTANCE_NORM, cudnn_norm_mode)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(din_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_MODE, + CUDNN_TYPE_NORM_MODE, + 1, + &cudnn_norm_mode)); + + // Set input tensor X + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Instancenorm_backward_attributes::input_names::X); + auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(din_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &x_desc)); + + // Set DY tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(DY, Instancenorm_backward_attributes::input_names::DY); + auto dy_desc = tensors.at(DY->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(din_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_DYDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dy_desc)); + + // Set scale tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SCALE, Instancenorm_backward_attributes::input_names::SCALE); + auto scale_desc = tensors.at(SCALE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(din_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_SCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &scale_desc)); + + // Set mean and inv_variance tensors + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(MEAN, Instancenorm_backward_attributes::input_names::MEAN); + auto mean_desc = tensors.at(MEAN->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(din_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_MEAN_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &mean_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(INV_VARIANCE, + Instancenorm_backward_attributes::input_names::INV_VARIANCE); + auto inv_var_desc = tensors.at(INV_VARIANCE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(din_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_INV_VARIANCE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &inv_var_desc)); + + // Set DSCALE and DBIAS output tensors + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DSCALE, Instancenorm_backward_attributes::output_names::DSCALE); + auto dscale_desc = tensors.at(DSCALE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(din_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_DSCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dscale_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DBIAS, Instancenorm_backward_attributes::output_names::DBIAS); + auto dbias_desc = tensors.at(DBIAS->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(din_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_DBIAS_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dbias_desc)); + + // Set DX output tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DX, Instancenorm_backward_attributes::output_names::DX); + auto dx_desc = tensors.at(DX->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(din_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_DXDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dx_desc)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(din_operation.get_raw_desc())); + + operations.push_back(std::make_shared<Operation_v8>(std::move(din_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "INSTANCE_NORM_BPROP"})"_json); + } +#endif +}; + +} // namespace graph + +} // namespace cudnn_frontend
\ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/layernorm.h b/deps/cudnn-frontend/include/cudnn_frontend/node/layernorm.h new file mode 100644 index 0000000..46420aa --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/layernorm.h @@ -0,0 +1,259 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend { + +namespace graph { +class LayerNormNode : public NodeCRTP<LayerNormNode> { + public: + Layernorm_attributes attributes; + + LayerNormNode(Layernorm_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::LAYERNORM; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for layernorm node " << attributes.name); + + attributes.fill_from_context(context); + + auto X = attributes.inputs[Layernorm_attributes::input_names::X]; + auto Y = attributes.outputs[Layernorm_attributes::output_names::Y]; + + // Only infer dims and strides if user did not set them + if (Y->get_dim().empty()) { + Y->set_dim(X->get_dim()); + } + if (Y->get_stride().empty()) { + Y->set_stride(X->get_stride()); + } + + // scale_bias dim is 1,c,h,w + auto scale_bias_dim = X->get_dim(); + scale_bias_dim[0] = 1; + + auto scale = attributes.inputs[Layernorm_attributes::input_names::SCALE]; + // Only infer dims and strides if user did not set them + if (scale->get_dim().empty()) { + scale->set_dim(scale_bias_dim); + } + if (scale->get_stride().empty()) { + auto const& scale_dim = scale->get_dim(); + std::vector<int64_t> stride_order; + CHECK_CUDNN_FRONTEND_ERROR( + detail::generate_stride_order_preserving_format(X->get_stride(), scale_dim.size(), stride_order)); + scale->set_stride(detail::generate_stride(scale_dim, stride_order)); + } + + auto bias = attributes.inputs[Layernorm_attributes::input_names::BIAS]; + // Only infer dims and strides if user did not set them + if (bias->get_dim().empty()) { + bias->set_dim(scale_bias_dim); + } + if (bias->get_stride().empty()) { + auto const& bias_dim = bias->get_dim(); + std::vector<int64_t> stride_order; + CHECK_CUDNN_FRONTEND_ERROR( + detail::generate_stride_order_preserving_format(X->get_stride(), bias_dim.size(), stride_order)); + bias->set_stride(detail::generate_stride(bias_dim, stride_order)); + } + + if (attributes.forward_phase == NormFwdPhase_t::TRAINING) { + // stats dim is x where scale == 1 else 1 + auto stats_dim = X->get_dim(); + for (size_t i = 0; i < stats_dim.size(); i++) { + if (scale->get_dim()[i] != 1) { + stats_dim[i] = 1; + } + } + + auto mean = attributes.outputs[Layernorm_attributes::output_names::MEAN]; + // Only infer dims and strides if user did not set them + if (mean->get_dim().empty()) { + mean->set_dim(stats_dim); + } + if (mean->get_stride().empty()) { + auto const& mean_dim = mean->get_dim(); + std::vector<int64_t> stride_order; + CHECK_CUDNN_FRONTEND_ERROR( + detail::generate_stride_order_preserving_format(X->get_stride(), mean_dim.size(), stride_order)); + mean->set_stride(detail::generate_stride(mean_dim, stride_order)); + } + + auto inv_var = attributes.outputs[Layernorm_attributes::output_names::INV_VARIANCE]; + // Only infer dims and strides if user did not set them + if (inv_var->get_dim().empty()) { + inv_var->set_dim(stats_dim); + } + if (inv_var->get_stride().empty()) { + auto const& inv_var_dim = inv_var->get_dim(); + std::vector<int64_t> stride_order; + CHECK_CUDNN_FRONTEND_ERROR( + detail::generate_stride_order_preserving_format(X->get_stride(), inv_var_dim.size(), stride_order)); + inv_var->set_stride(detail::generate_stride(inv_var_dim, stride_order)); + } + } + + // Set scalar tensors + std::vector<int64_t> ones(X->get_dim().size(), 1); + auto infer_scalar_tensors = [&ones](std::shared_ptr<Tensor_attributes>& T) { + // Only infer dims and strides if user did not set them + if (T->get_dim().empty()) { + T->set_dim(ones); + } + if (T->get_stride().empty()) { + T->set_stride(ones); + } + }; + infer_scalar_tensors(attributes.inputs[Layernorm_attributes::input_names::EPSILON]); + + return {error_code_t::OK, ""}; + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: " << "Validating LayerNormNode " << attributes.name); + + // Norm forward phase should be set + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.forward_phase == NormFwdPhase_t::NOT_SET, + error_code_t::ATTRIBUTE_NOT_SET, + "Forward phase not set of layernorm node."); + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations, + std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: " << "Building LayerNormNode operations " << attributes.name << " "); + + // Create operation by directly calling cuDNN backend API + Operation_v8 layernorm_operation; + + _CUDNN_CHECK_CUDNN_ERROR( + layernorm_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_NORM_FORWARD_DESCRIPTOR)); + + // Set norm mode + cudnnBackendNormMode_t cudnn_norm_mode; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormMode_t::LAYER_NORM, cudnn_norm_mode)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(layernorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_MODE, + CUDNN_TYPE_NORM_MODE, + 1, + &cudnn_norm_mode)); + + // Set forward phase + cudnnBackendNormFwdPhase_t cudnn_norm_fwd_phase; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.forward_phase, cudnn_norm_fwd_phase)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(layernorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_PHASE, + CUDNN_TYPE_NORM_FWD_PHASE, + 1, + &cudnn_norm_fwd_phase)); + + // Set input tensor X + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Layernorm_attributes::input_names::X); + auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(layernorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &x_desc)); + + // Set scale and bias tensors + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SCALE, Layernorm_attributes::input_names::SCALE); + auto scale_desc = tensors.at(SCALE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(layernorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_SCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &scale_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(BIAS, Layernorm_attributes::input_names::BIAS); + auto bias_desc = tensors.at(BIAS->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(layernorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_BIAS_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &bias_desc)); + + // Set epsilon tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(EPSILON, Layernorm_attributes::input_names::EPSILON); + auto epsilon_desc = tensors.at(EPSILON->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(layernorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_EPSILON_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &epsilon_desc)); + + // Set output tensor Y + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(Y, Layernorm_attributes::output_names::Y); + auto y_desc = tensors.at(Y->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(layernorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_YDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &y_desc)); + + // Set mean and inv_variance for training phase + if (attributes.forward_phase == NormFwdPhase_t::TRAINING) { + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(MEAN, Layernorm_attributes::output_names::MEAN); + auto mean_desc = tensors.at(MEAN->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(layernorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_MEAN_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &mean_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(INV_VARIANCE, Layernorm_attributes::output_names::INV_VARIANCE); + auto inv_var_desc = tensors.at(INV_VARIANCE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(layernorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_INV_VARIANCE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &inv_var_desc)); + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(layernorm_operation.get_raw_desc())); + + operations.push_back(std::make_shared<Operation_v8>(std::move(layernorm_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "LAYER_NORM"})"_json); + } +#endif +}; + +} // namespace graph + +} // namespace cudnn_frontend
\ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/matmul.h b/deps/cudnn-frontend/include/cudnn_frontend/node/matmul.h new file mode 100644 index 0000000..f09d341 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/matmul.h @@ -0,0 +1,253 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend::graph { + +class MatmulNode : public NodeCRTP<MatmulNode> { + public: + Matmul_attributes attributes; + + MatmulNode(Matmul_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::MATMUL; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for matmul node " << attributes.name); + + attributes.fill_from_context(context); + + // Only inferrencing from (A, B) -> C works today. + auto a_tensor = attributes.inputs[Matmul_attributes::input_names::A]; + auto b_tensor = attributes.inputs[Matmul_attributes::input_names::B]; + auto c_tensor = attributes.outputs[Matmul_attributes::output_names::C]; + + auto const a_tensor_dim = a_tensor->get_dim(); + auto const b_tensor_dim = b_tensor->get_dim(); + auto c_tensor_dim = c_tensor->get_dim(); + + // Only infer dims and strides if user did not set them + if (c_tensor_dim.empty()) { + // CHECK_CUDNN_FRONTEND_ERROR(detail::generate_matmul_output_dim(a_tensor_dim, b_tensor_dim, c_tensor_dim)); + + c_tensor_dim.resize(a_tensor_dim.size()); + int64_t gemm_start_dim = a_tensor_dim.size() - 2; + c_tensor_dim[gemm_start_dim] = a_tensor_dim[gemm_start_dim]; // M + c_tensor_dim[gemm_start_dim + 1] = b_tensor_dim[gemm_start_dim + 1]; // N + + // Broadcast the batches + for (int64_t i = 0; i < gemm_start_dim; ++i) { + c_tensor_dim[i] = std::max(a_tensor_dim[i], b_tensor_dim[i]); + } + + c_tensor->set_dim(c_tensor_dim); + } + if (c_tensor->get_stride().empty()) { + auto const& c_dim = c_tensor->get_dim(); + // Default to Col major + auto const& stride_order = detail::generate_row_major_stride_order(c_dim.size()); + c_tensor->set_stride(detail::generate_stride(c_dim, stride_order)); + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations, + std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: " << "Building MatmulNode operations " << attributes.name << " "); + + // Create matmul descriptor by directly calling cuDNN backend API + MatMulDesc_v8 matmul_descriptor; + + _CUDNN_CHECK_CUDNN_ERROR(matmul_descriptor.initialize_managed_backend_pointer(CUDNN_BACKEND_MATMUL_DESCRIPTOR)); + + // Set compute type + cudnnDataType_t cudnn_data_type; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.compute_data_type, cudnn_data_type)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute( + matmul_descriptor.get_raw_desc(), CUDNN_ATTR_MATMUL_COMP_TYPE, CUDNN_TYPE_DATA_TYPE, 1, &cudnn_data_type)); + + // Set padding value if specified +#if (CUDNN_VERSION >= 8900) + if (attributes.padding_value != 0.0) { + double padding_value = attributes.padding_value; + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(matmul_descriptor.get_raw_desc(), + CUDNN_ATTR_MATMUL_PADDING_VALUE, + CUDNN_TYPE_DOUBLE, + 1, + &padding_value)); + } +#endif + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(matmul_descriptor.get_raw_desc())); + CUDNN_FE_LOG_LABEL_ENDL(matmul_descriptor); + + // Create operation by directly calling cuDNN backend API + Operation_v8 matmul_operation; + + _CUDNN_CHECK_CUDNN_ERROR( + matmul_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_MATMUL_DESCRIPTOR)); + + // Set input tensor A + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(A, Matmul_attributes::input_names::A); + auto a_desc = tensors.at(A->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(matmul_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_MATMUL_ADESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &a_desc)); + + // Set input tensor B + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(B, Matmul_attributes::input_names::B); + auto b_desc = tensors.at(B->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(matmul_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_MATMUL_BDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &b_desc)); + + // Set output tensor C + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(C, Matmul_attributes::output_names::C); + auto c_desc = tensors.at(C->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(matmul_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_MATMUL_CDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &c_desc)); + + // Set matmul descriptor + auto matmul_desc_ptr = matmul_descriptor.get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(matmul_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_MATMUL_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &matmul_desc_ptr)); + + // Set optional override tensors + auto M_override = attributes.inputs.find(Matmul_attributes::input_names::M_override); + if ((M_override != attributes.inputs.end()) && (M_override->second != nullptr)) { + auto m_override_desc = tensors.at(M_override->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(matmul_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_MATMUL_GEMM_M_OVERRIDE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &m_override_desc)); + } + + auto N_override = attributes.inputs.find(Matmul_attributes::input_names::N_override); + if ((N_override != attributes.inputs.end()) && (N_override->second != nullptr)) { + auto n_override_desc = tensors.at(N_override->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(matmul_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_MATMUL_GEMM_N_OVERRIDE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &n_override_desc)); + } + + auto K_override = attributes.inputs.find(Matmul_attributes::input_names::K_override); + if ((K_override != attributes.inputs.end()) && (K_override->second != nullptr)) { + auto k_override_desc = tensors.at(K_override->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(matmul_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_MATMUL_GEMM_K_OVERRIDE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &k_override_desc)); + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(matmul_operation.get_raw_desc())); + + operations.push_back(std::make_shared<Operation_v8>(std::move(matmul_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "MATMUL"})"_json); + } +#endif +}; + +inline void +INode::matmul(std::shared_ptr<Tensor_attributes> a, + std::shared_ptr<Tensor_attributes> b, + Matmul_attributes attributes, + std::shared_ptr<Tensor_attributes> c) { + attributes.inputs[Matmul_attributes::input_names::A] = a; + attributes.inputs[Matmul_attributes::input_names::B] = b; + attributes.outputs[Matmul_attributes::output_names::C] = c; + sub_nodes.emplace_back(std::make_unique<MatmulNode>(std::move(attributes), context)); +} + +inline std::shared_ptr<Tensor_attributes> +INode::matmul(std::shared_ptr<Tensor_attributes> a, + std::shared_ptr<Tensor_attributes> b, + Matmul_attributes attributes) { + if (attributes.name.empty()) { + attributes.name += std::to_string(sub_nodes.size()); + } + attributes.inputs[Matmul_attributes::input_names::A] = a; + attributes.inputs[Matmul_attributes::input_names::B] = b; + + if (a->get_name().empty()) { + a->set_name(attributes.name + "::A"); + }; + if (b->get_name().empty()) { + b->set_name(attributes.name + "::B"); + }; + + auto m_override = attributes.inputs.find(Matmul_attributes::input_names::M_override); + auto n_override = attributes.inputs.find(Matmul_attributes::input_names::N_override); + auto k_override = attributes.inputs.find(Matmul_attributes::input_names::K_override); + + if (m_override != attributes.inputs.end()) { + auto tensor = m_override->second; + if (tensor && tensor->get_name().empty()) { + tensor->set_name(attributes.name + "::M_override"); + } + } + if (n_override != attributes.inputs.end()) { + auto tensor = n_override->second; + if (tensor && tensor->get_name().empty()) { + tensor->set_name(attributes.name + "::N_override"); + } + } + if (k_override != attributes.inputs.end()) { + auto tensor = k_override->second; + if (tensor && tensor->get_name().empty()) { + tensor->set_name(attributes.name + "::K_override"); + } + } + + auto C = attributes.outputs[Matmul_attributes::output_names::C] = output_tensor(attributes.name + "::C"); + + sub_nodes.emplace_back(std::make_unique<MatmulNode>(std::move(attributes), context)); + return C; +} + +} // namespace cudnn_frontend::graph
\ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/matmul_fp8.h b/deps/cudnn-frontend/include/cudnn_frontend/node/matmul_fp8.h new file mode 100644 index 0000000..d3fe5e5 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/matmul_fp8.h @@ -0,0 +1,104 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend::graph { + +class MatmulFP8Node : public NodeCRTP<MatmulFP8Node> { + public: + Matmul_fp8_attributes attributes; + + MatmulFP8Node(Matmul_fp8_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::MATMUL; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for matmul fp8 node " << attributes.name); + + attributes.fill_from_context(context); + + auto const& a_dim = attributes.inputs.at(Matmul_fp8_attributes::input_names::A)->get_dim(); + auto const& b_dim = attributes.inputs.at(Matmul_fp8_attributes::input_names::B)->get_dim(); + auto const& c_dim = attributes.outputs.at(Matmul_fp8_attributes::output_names::C)->get_dim(); + + std::shared_ptr<Tensor_attributes> last_output; + + // Matmul + + auto matmul_attributes = Matmul_attributes(); + matmul_attributes.clone_fp8_attributes(attributes); + matmul_attributes.set_name("matmul"); + + last_output = matmul(attributes.inputs.at(Matmul_fp8_attributes::input_names::A), + attributes.inputs.at(Matmul_fp8_attributes::input_names::B), + matmul_attributes); + + // Reduction if GQA for head dimension + if (a_dim.size() == 4 && b_dim.size() == 4 && c_dim.size() == 4 && a_dim[1] == b_dim[1] && + a_dim[1] != c_dim[1] && (a_dim[1] % c_dim[1] == 0)) { + auto gqa_attributes = Reduction_attributes().set_name("gqa_c").set_mode(ReductionMode_t::ADD); + last_output = reduction(last_output, gqa_attributes); + last_output->set_dim(c_dim); + } + + //// Scale Descales + auto mul_attributes = Pointwise_attributes().set_mode(PointwiseMode_t::MUL); + // Descale A + mul_attributes.set_name("descale_a"); + last_output = + pointwise(last_output, attributes.inputs.at(Matmul_fp8_attributes::input_names::Descale_A), mul_attributes); + + // Descale B + mul_attributes.set_name("descale_b"); + last_output = + pointwise(last_output, attributes.inputs.at(Matmul_fp8_attributes::input_names::Descale_B), mul_attributes); + + // Scale C + mul_attributes.set_name("scale_c"); + // Special non-functional-style call. Needed because output already created and provided to user. + pointwise(last_output, + attributes.inputs.at(Matmul_fp8_attributes::input_names::Scale_C), + mul_attributes, + attributes.outputs.at(Matmul_fp8_attributes::output_names::C)); + + // Amax C + auto amax_attributes = Reduction_attributes().set_name("amax_c").set_mode(ReductionMode_t::AMAX); + // Special non-functional-style call. Needed because output already created and provided to user. + reduction(last_output, amax_attributes, attributes.outputs.at(Matmul_fp8_attributes::output_names::Amax_C)); + + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "MATMUL_FP8"})"_json); + } +#endif +}; +inline void +INode::matmul_fp8(std::shared_ptr<Tensor_attributes> a, + std::shared_ptr<Tensor_attributes> b, + std::shared_ptr<Tensor_attributes> descale_a, + std::shared_ptr<Tensor_attributes> descale_b, + std::shared_ptr<Tensor_attributes> scale_c, + Matmul_fp8_attributes attributes, + std::shared_ptr<Tensor_attributes> c, + std::shared_ptr<Tensor_attributes> amax_c) { + attributes.inputs[Matmul_fp8_attributes::input_names::A] = a; + attributes.inputs[Matmul_fp8_attributes::input_names::B] = b; + attributes.inputs[Matmul_fp8_attributes::input_names::Descale_A] = descale_a; + attributes.inputs[Matmul_fp8_attributes::input_names::Descale_B] = descale_b; + attributes.inputs[Matmul_fp8_attributes::input_names::Scale_C] = scale_c; + attributes.outputs[Matmul_fp8_attributes::output_names::C] = c; + attributes.outputs[Matmul_fp8_attributes::output_names::Amax_C] = amax_c; + sub_nodes.emplace_back(std::make_unique<MatmulFP8Node>(std::move(attributes), context)); +} +} // namespace cudnn_frontend::graph
\ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/moe_grouped_matmul.h b/deps/cudnn-frontend/include/cudnn_frontend/node/moe_grouped_matmul.h new file mode 100644 index 0000000..d05d75d --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/moe_grouped_matmul.h @@ -0,0 +1,221 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend::graph { + +class MoeGroupedMatmulNode : public NodeCRTP<MoeGroupedMatmulNode> { + public: + Moe_grouped_matmul_attributes attributes; + + MoeGroupedMatmulNode(Moe_grouped_matmul_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::MOE_GROUPED_MATMUL; + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating MoeGroupedMatmulNode " << attributes.name); + + auto const token_it = attributes.inputs.find(Moe_grouped_matmul_attributes::input_names::Token); + RETURN_CUDNN_FRONTEND_ERROR_IF(token_it == attributes.inputs.end() || token_it->second == nullptr, + error_code_t::ATTRIBUTE_NOT_SET, + "MoeGroupedMatmul input Token not set."); + + auto const weight_it = attributes.inputs.find(Moe_grouped_matmul_attributes::input_names::Weight); + RETURN_CUDNN_FRONTEND_ERROR_IF(weight_it == attributes.inputs.end() || weight_it->second == nullptr, + error_code_t::ATTRIBUTE_NOT_SET, + "MoeGroupedMatmul input Weight not set."); + + auto const first_token_offset_it = + attributes.inputs.find(Moe_grouped_matmul_attributes::input_names::FirstTokenOffset); + RETURN_CUDNN_FRONTEND_ERROR_IF( + first_token_offset_it == attributes.inputs.end() || first_token_offset_it->second == nullptr, + error_code_t::ATTRIBUTE_NOT_SET, + "MoeGroupedMatmul input FirstTokenOffset not set."); + + auto const output_it = attributes.outputs.find(Moe_grouped_matmul_attributes::output_names::Output); + RETURN_CUDNN_FRONTEND_ERROR_IF(output_it == attributes.outputs.end() || output_it->second == nullptr, + error_code_t::ATTRIBUTE_NOT_SET, + "MoeGroupedMatmul output Output not set."); + + return {error_code_t::OK, ""}; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for moe grouped matmul node " << attributes.name); + + attributes.fill_from_context(context); + + auto token_tensor = attributes.inputs[Moe_grouped_matmul_attributes::input_names::Token]; + auto weight_tensor = attributes.inputs[Moe_grouped_matmul_attributes::input_names::Weight]; + auto token_index_tensor = attributes.inputs[Moe_grouped_matmul_attributes::input_names::TokenIndex]; + auto output_tensor = attributes.outputs[Moe_grouped_matmul_attributes::output_names::Output]; + + auto const token_tensor_dim = token_tensor->get_dim(); + auto const weight_tensor_dim = weight_tensor->get_dim(); + auto output_tensor_dim = output_tensor->get_dim(); + + if (output_tensor_dim.empty()) { + output_tensor_dim.resize(3); + output_tensor_dim[0] = 1; + output_tensor_dim[2] = weight_tensor_dim[2]; + if (attributes.mode == MoeGroupedMatmulMode_t::GATHER) { + output_tensor_dim[1] = token_index_tensor->get_dim()[1]; + } else { + output_tensor_dim[1] = token_tensor_dim[1]; + } + output_tensor_dim.resize(3); + + output_tensor->set_dim(output_tensor_dim); + } + + if (output_tensor->get_stride().empty()) { + auto const& output_dim = output_tensor->get_dim(); + auto const& stride_order = detail::generate_row_major_stride_order(output_dim.size()); + output_tensor->set_stride(detail::generate_stride(output_dim, stride_order)); + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations, + std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final { + getLogger() << "[cudnn_frontend] INFO: " + << "Building MoeGroupedMatmulNode operations " << attributes.name << std::endl; + auto cudnn_ver_error = error_t{error_code_t::GRAPH_NOT_SUPPORTED, "Moe grouped matmul requires cuDNN v9.15.0"}; + +#if (CUDNN_VERSION >= 91500) + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(91500, cudnn_ver_error); + CUDNN_FRONTEND_UNUSED(operations); + + auto moe_grouped_matmul_operation = + make_shared_backend_pointer(CUDNN_BACKEND_OPERATION_MOE_GROUPED_MATMUL_DESCRIPTOR); + + cudnnMoeGroupedMatmulMode_t moe_grouped_matmul_mode; + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.mode, moe_grouped_matmul_mode)); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_MODE, + CUDNN_TYPE_MOE_GROUPED_MATMUL_MODE, + 1, + &moe_grouped_matmul_mode)); + + cudnnDataType_t cudnn_data_type; + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.compute_data_type, cudnn_data_type)); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_MATH_PREC, + CUDNN_TYPE_DATA_TYPE, + 1, + &cudnn_data_type)); + + auto token = attributes.inputs.find(Moe_grouped_matmul_attributes::input_names::Token)->second; + auto backend_token = tensors[token->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_TOKEN_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_token)); + + auto weight = attributes.inputs.find(Moe_grouped_matmul_attributes::input_names::Weight)->second; + auto backend_weight = tensors[weight->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_WEIGHT_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_weight)); + + auto first_token_offset = + attributes.inputs.find(Moe_grouped_matmul_attributes::input_names::FirstTokenOffset)->second; + auto backend_first_token_offset = tensors[first_token_offset->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_FIRST_TOKEN_OFFSET_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_first_token_offset)); + + auto output = attributes.outputs.find(Moe_grouped_matmul_attributes::output_names::Output)->second; + auto backend_output = tensors[output->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_OUTPUT_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_output)); + + if (attributes.mode == MoeGroupedMatmulMode_t::GATHER || attributes.mode == MoeGroupedMatmulMode_t::SCATTER) { + auto token_index = attributes.inputs.find(Moe_grouped_matmul_attributes::input_names::TokenIndex)->second; + auto backend_token_index = tensors[token_index->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_TOKEN_INDEX_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_token_index)); + } + + if (attributes.mode == MoeGroupedMatmulMode_t::SCATTER) { + auto token_ks = attributes.inputs.find(Moe_grouped_matmul_attributes::input_names::TokenKs)->second; + auto backend_token_ks = tensors[token_ks->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_TOKEN_KS_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_token_ks)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_TOP_K, + CUDNN_TYPE_INT32, + 1, + &(attributes.top_k))); + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(moe_grouped_matmul_operation->get_backend_descriptor())); + + raw_operations.push_back(moe_grouped_matmul_operation); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; +#else + CUDNN_FRONTEND_UNUSED(uids_involved_in_operations); + CUDNN_FRONTEND_UNUSED(operations); + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FRONTEND_UNUSED(tensors); + return cudnn_ver_error; +#endif + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "MOE_GROUPED_MATMUL"})"_json); + } +#endif +}; + +inline void +INode::moe_grouped_matmul(std::shared_ptr<Tensor_attributes> token, + std::shared_ptr<Tensor_attributes> weight, + std::shared_ptr<Tensor_attributes> first_token_offset, + std::shared_ptr<Tensor_attributes> token_index, + std::shared_ptr<Tensor_attributes> token_ks, + Moe_grouped_matmul_attributes attributes, + std::shared_ptr<Tensor_attributes> output) { + attributes.inputs[Moe_grouped_matmul_attributes::input_names::Token] = token; + attributes.inputs[Moe_grouped_matmul_attributes::input_names::Weight] = weight; + attributes.inputs[Moe_grouped_matmul_attributes::input_names::FirstTokenOffset] = first_token_offset; + attributes.inputs[Moe_grouped_matmul_attributes::input_names::TokenIndex] = token_index; + attributes.inputs[Moe_grouped_matmul_attributes::input_names::TokenKs] = token_ks; + attributes.outputs[Moe_grouped_matmul_attributes::output_names::Output] = output; + sub_nodes.emplace_back(std::make_unique<MoeGroupedMatmulNode>(std::move(attributes), context)); +} + +} // namespace cudnn_frontend::graph
\ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/moe_grouped_matmul_bwd.h b/deps/cudnn-frontend/include/cudnn_frontend/node/moe_grouped_matmul_bwd.h new file mode 100644 index 0000000..2076203 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/moe_grouped_matmul_bwd.h @@ -0,0 +1,181 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend::graph { + +class MoeGroupedMatmulBwdNode : public NodeCRTP<MoeGroupedMatmulBwdNode> { + public: + Moe_grouped_matmul_bwd_attributes attributes; + + MoeGroupedMatmulBwdNode(Moe_grouped_matmul_bwd_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::MOE_GROUPED_MATMUL_BWD; + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating MoeGroupedMatmulBwdNode " << attributes.name); + + auto const doutput_it = attributes.inputs.find(Moe_grouped_matmul_bwd_attributes::input_names::DOutput); + RETURN_CUDNN_FRONTEND_ERROR_IF(doutput_it == attributes.inputs.end() || doutput_it->second == nullptr, + error_code_t::ATTRIBUTE_NOT_SET, + "MoeGroupedMatmulBwd input DOutput not set."); + + auto const token_it = attributes.inputs.find(Moe_grouped_matmul_bwd_attributes::input_names::Token); + RETURN_CUDNN_FRONTEND_ERROR_IF(token_it == attributes.inputs.end() || token_it->second == nullptr, + error_code_t::ATTRIBUTE_NOT_SET, + "MoeGroupedMatmulBwd input Token not set."); + + auto const first_token_offset_it = + attributes.inputs.find(Moe_grouped_matmul_bwd_attributes::input_names::FirstTokenOffset); + RETURN_CUDNN_FRONTEND_ERROR_IF( + first_token_offset_it == attributes.inputs.end() || first_token_offset_it->second == nullptr, + error_code_t::ATTRIBUTE_NOT_SET, + "MoeGroupedMatmulBwd input FirstTokenOffset not set."); + + auto const dweight_it = attributes.outputs.find(Moe_grouped_matmul_bwd_attributes::output_names::DWeight); + RETURN_CUDNN_FRONTEND_ERROR_IF(dweight_it == attributes.outputs.end() || dweight_it->second == nullptr, + error_code_t::ATTRIBUTE_NOT_SET, + "MoeGroupedMatmulBwd output DWeight not set."); + + return {error_code_t::OK, ""}; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for moe grouped matmul bwd node " + << attributes.name); + + attributes.fill_from_context(context); + + auto doutput_tensor = attributes.inputs[Moe_grouped_matmul_bwd_attributes::input_names::DOutput]; + auto token_tensor = attributes.inputs[Moe_grouped_matmul_bwd_attributes::input_names::Token]; + auto token_index_tensor = attributes.inputs[Moe_grouped_matmul_bwd_attributes::input_names::FirstTokenOffset]; + auto dweight_tensor = attributes.outputs[Moe_grouped_matmul_bwd_attributes::output_names::DWeight]; + + auto const doutput_tensor_dim = doutput_tensor->get_dim(); + auto const token_tensor_dim = token_tensor->get_dim(); + auto const token_index_tensor_dim = token_index_tensor->get_dim(); + auto dweight_tensor_dim = dweight_tensor->get_dim(); + + if (dweight_tensor_dim.empty()) { + dweight_tensor_dim.resize(3); + dweight_tensor_dim[0] = token_index_tensor_dim[0]; + dweight_tensor_dim[1] = token_tensor_dim[2]; + dweight_tensor_dim[2] = doutput_tensor_dim[2]; + dweight_tensor->set_dim(dweight_tensor_dim); + } + + if (dweight_tensor->get_stride().empty()) { + auto const& dweight_dim = dweight_tensor->get_dim(); + auto const& stride_order = detail::generate_column_major_stride_order(dweight_dim.size()); + dweight_tensor->set_stride(detail::generate_stride(dweight_dim, stride_order)); + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations, + std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final { + getLogger() << "[cudnn_frontend] INFO: " + << "Building MoeGroupedMatmulBwdNode operations " << attributes.name << std::endl; + auto cudnn_ver_error = + error_t{error_code_t::GRAPH_NOT_SUPPORTED, "Moe grouped matmul bwd requires cuDNN v9.22.0"}; + +#if (CUDNN_VERSION >= 92200) + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(92200, cudnn_ver_error); + CUDNN_FRONTEND_UNUSED(operations); + + auto moe_grouped_matmul_bwd_operation = + make_shared_backend_pointer(CUDNN_BACKEND_OPERATION_MOE_GROUPED_MATMUL_BWD_DESCRIPTOR); + + cudnnDataType_t cudnn_data_type; + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.compute_data_type, cudnn_data_type)); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_bwd_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_BWD_MATH_PREC, + CUDNN_TYPE_DATA_TYPE, + 1, + &cudnn_data_type)); + + auto token = attributes.inputs.find(Moe_grouped_matmul_bwd_attributes::input_names::Token)->second; + auto backend_token = tensors[token->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_bwd_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_BWD_TOKEN_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_token)); + + auto dweight = attributes.outputs.find(Moe_grouped_matmul_bwd_attributes::output_names::DWeight)->second; + auto backend_dweight = tensors[dweight->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_bwd_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_BWD_DWEIGHT_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_dweight)); + + auto first_token_offset = + attributes.inputs.find(Moe_grouped_matmul_bwd_attributes::input_names::FirstTokenOffset)->second; + auto backend_first_token_offset = tensors[first_token_offset->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR( + detail::set_attribute(moe_grouped_matmul_bwd_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_BWD_FIRST_TOKEN_OFFSET_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_first_token_offset)); + + auto doutput = attributes.inputs.find(Moe_grouped_matmul_bwd_attributes::input_names::DOutput)->second; + auto backend_doutput = tensors[doutput->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_bwd_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_BWD_DOUTPUT_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_doutput)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(moe_grouped_matmul_bwd_operation->get_backend_descriptor())); + + raw_operations.push_back(moe_grouped_matmul_bwd_operation); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; +#else + CUDNN_FRONTEND_UNUSED(uids_involved_in_operations); + CUDNN_FRONTEND_UNUSED(operations); + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FRONTEND_UNUSED(tensors); + return cudnn_ver_error; +#endif + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "MOE_GROUPED_MATMUL_BWD"})"_json); + } +#endif +}; + +inline void +INode::moe_grouped_matmul_bwd(std::shared_ptr<Tensor_attributes> doutput, + std::shared_ptr<Tensor_attributes> token, + std::shared_ptr<Tensor_attributes> first_token_offset, + Moe_grouped_matmul_bwd_attributes attributes, + std::shared_ptr<Tensor_attributes> dweight) { + attributes.inputs[Moe_grouped_matmul_bwd_attributes::input_names::DOutput] = doutput; + attributes.inputs[Moe_grouped_matmul_bwd_attributes::input_names::Token] = token; + attributes.inputs[Moe_grouped_matmul_bwd_attributes::input_names::FirstTokenOffset] = first_token_offset; + attributes.outputs[Moe_grouped_matmul_bwd_attributes::output_names::DWeight] = dweight; + sub_nodes.emplace_back(std::make_unique<MoeGroupedMatmulBwdNode>(std::move(attributes), context)); +} + +} // namespace cudnn_frontend::graph
\ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/paged_cache_load.h b/deps/cudnn-frontend/include/cudnn_frontend/node/paged_cache_load.h new file mode 100644 index 0000000..9fc8610 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/paged_cache_load.h @@ -0,0 +1,153 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +#include "pointwise.h" +#include "reduction.h" + +namespace cudnn_frontend::graph { + +class PagedCacheLoadNode : public NodeCRTP<PagedCacheLoadNode> { + public: + PagedCacheLoad_attributes attributes; + + PagedCacheLoadNode(PagedCacheLoad_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::PAGED_CACHE_LOAD; + } + + error_t + create_cudnn_operations( + std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations, + std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: " << "Building PagedCacheLoadNode operations " << attributes.name << " "); + auto cudnn_ver_error = error_t{error_code_t::GRAPH_NOT_SUPPORTED, "Paged cache load requires cuDNN v9.5.0"}; + +#if (CUDNN_VERSION >= 90500) + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(90500, cudnn_ver_error); + + // Create operation by directly calling cuDNN backend API + Operation_v8 paged_cache_load_operation; + + _CUDNN_CHECK_CUDNN_ERROR(paged_cache_load_operation.initialize_managed_backend_pointer( + CUDNN_BACKEND_OPERATION_PAGED_CACHE_LOAD_DESCRIPTOR)); + + // Set container tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(container, PagedCacheLoad_attributes::input_names::container); + auto container_desc = tensors.at(container->second->get_uid())->get_raw_desc(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(paged_cache_load_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_PAGED_CACHE_LOAD_CONTAINER_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &container_desc)); + + // Set page table tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(pageTable, PagedCacheLoad_attributes::input_names::pageTable); + auto page_table_desc = tensors.at(pageTable->second->get_uid())->get_raw_desc(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(paged_cache_load_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_PAGED_CACHE_LOAD_PAGE_TABLE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &page_table_desc)); + + // Set sequence length tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(seqLen, PagedCacheLoad_attributes::input_names::seqLen); + auto seq_len_desc = tensors.at(seqLen->second->get_uid())->get_raw_desc(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(paged_cache_load_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_PAGED_CACHE_LOAD_SEQUENCE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &seq_len_desc)); + + // Set output tensor Y + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(yOut, PagedCacheLoad_attributes::output_names::yOut); + auto y_desc = tensors.at(yOut->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(paged_cache_load_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_PAGED_CACHE_LOAD_YDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &y_desc)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(paged_cache_load_operation.get_raw_desc())); + + operations.push_back(std::make_shared<Operation_v8>(std::move(paged_cache_load_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + + return {error_code_t::OK, ""}; +#else + CUDNN_FRONTEND_UNUSED(uids_involved_in_operations); + CUDNN_FRONTEND_UNUSED(operations); + CUDNN_FRONTEND_UNUSED(tensors); + return cudnn_ver_error; +#endif + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating PagedCacheLoadNode " << attributes.name); + + RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 90500 || detail::get_compiled_version() < 90500, + error_code_t::CUDNN_BACKEND_API_FAILED, + "The cuDNN backend version must be at least 9.5.0 at compile time and runtime " + "in order to use PagedCacheLoadNode."); + + auto const yOut_dims = attributes.outputs.at(PagedCacheLoad_attributes::output_names::yOut)->get_dim(); + auto const yOut_strides = attributes.outputs.at(PagedCacheLoad_attributes::output_names::yOut)->get_stride(); + auto const container_dims = attributes.inputs.at(PagedCacheLoad_attributes::input_names::container)->get_dim(); + auto const blockTable_dims = attributes.inputs.at(PagedCacheLoad_attributes::input_names::pageTable)->get_dim(); + + // In the backend, the k-cache is passed as K^T and has dims [B,H,D,S], while v-cache has dims [B,H,S,D] + // Use the strides to distinguish. + auto yIsTransposed = yOut_strides[2] == 1; + auto s_kv = !yIsTransposed ? yOut_dims[2] : yOut_dims[3]; + + auto block_size = container_dims[2]; + auto block_table_size = blockTable_dims[2]; + bool is_block_table_packed = + attributes.inputs.at(PagedCacheLoad_attributes::input_names::pageTable)->get_ragged_offset() != nullptr; + + RETURN_CUDNN_FRONTEND_ERROR_IF( + !is_block_table_packed && (s_kv + (block_size - 1)) / block_size != block_table_size, + error_code_t::INVALID_VALUE, + "Paged cache load: block table size must equal ceil(s_kv/block_size), except when using packed block " + "tables"); + + return {error_code_t::OK, ""}; + } + + error_t + infer_properties_node() override final { + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + } +#endif +}; + +inline void +INode::paged_cache_load(std::shared_ptr<Tensor_attributes> container, + std::shared_ptr<Tensor_attributes> seqLen, + std::shared_ptr<Tensor_attributes> pageTable, + PagedCacheLoad_attributes attributes, + std::shared_ptr<Tensor_attributes> yOut) { + attributes.inputs[PagedCacheLoad_attributes::input_names::container] = std::move(container); + attributes.inputs[PagedCacheLoad_attributes::input_names::seqLen] = std::move(seqLen); + attributes.inputs[PagedCacheLoad_attributes::input_names::pageTable] = std::move(pageTable); + attributes.outputs[PagedCacheLoad_attributes::output_names::yOut] = std::move(yOut); + sub_nodes.emplace_back(std::make_unique<PagedCacheLoadNode>(std::move(attributes), context)); +} +} // namespace cudnn_frontend::graph
\ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/pointwise.h b/deps/cudnn-frontend/include/cudnn_frontend/node/pointwise.h new file mode 100644 index 0000000..6d2da14 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/pointwise.h @@ -0,0 +1,390 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend::graph { + +class PointwiseNode : public NodeCRTP<PointwiseNode> { + public: + Pointwise_attributes attributes; + + PointwiseNode(Pointwise_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::POINTWISE; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for pointwise node " << attributes.name); + + attributes.fill_from_context(context); + + auto out_0_tensor = attributes.outputs.at(Pointwise_attributes::output_names::OUT_0); + + auto output_dim = out_0_tensor->get_dim(); + // Only infer dims and strides if user did not set them + if (output_dim.empty()) { + std::vector<std::vector<int64_t>> input_shapes; + for (const auto& [input_name, input_tensor] : attributes.inputs) { + if (!input_tensor) { + continue; + } + input_shapes.push_back(input_tensor->get_dim()); + } + + CHECK_CUDNN_FRONTEND_ERROR(detail::compute_broadcast_shape(input_shapes, output_dim)); + out_0_tensor->set_dim(output_dim); + } + + if (out_0_tensor->get_stride().empty()) { + for (const auto& [input_name, input_tensor] : attributes.inputs) { + if (input_tensor == nullptr) { + continue; + } + if (input_tensor->get_dim() == out_0_tensor->get_dim()) { + CUDNN_FE_LOG_LABEL_ENDL("INFO:" << " " << out_0_tensor->get_name() + << " stride computed from " << input_tensor->get_name()); + out_0_tensor->set_stride(input_tensor->get_stride()); + break; + } + } + if (out_0_tensor->get_stride().empty() && out_0_tensor->get_is_virtual()) { + // If the tensor is virtual the strides are immaterial + auto input_stride = attributes.inputs.at(Pointwise_attributes::input_names::IN_0)->get_stride(); + std::vector<int64_t> stride_order; + CHECK_CUDNN_FRONTEND_ERROR( + detail::generate_stride_order_preserving_format(input_stride, output_dim.size(), stride_order)); + out_0_tensor->set_stride(detail::generate_stride(output_dim, stride_order)); + } + RETURN_CUDNN_FRONTEND_ERROR_IF(out_0_tensor->get_stride().empty(), + error_code_t::SHAPE_DEDUCTION_FAILED, + "Pointwise output strides could not be computed"); + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations, + std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: " << "Building PointwiseNode operations " << attributes.name << " "); + + // Create pointwise descriptor by directly calling cuDNN backend API + PointWiseDesc_v8 pointwise_descriptor; + + _CUDNN_CHECK_CUDNN_ERROR( + pointwise_descriptor.initialize_managed_backend_pointer(CUDNN_BACKEND_POINTWISE_DESCRIPTOR)); + + // Set pointwise mode + cudnnPointwiseMode_t cudnn_pointwise_mode; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.mode, cudnn_pointwise_mode)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_descriptor.get_raw_desc(), + CUDNN_ATTR_POINTWISE_MODE, + CUDNN_TYPE_POINTWISE_MODE, + 1, + &cudnn_pointwise_mode)); + + // Set compute type + cudnnDataType_t cudnn_data_type; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.compute_data_type, cudnn_data_type)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_descriptor.get_raw_desc(), + CUDNN_ATTR_POINTWISE_MATH_PREC, + CUDNN_TYPE_DATA_TYPE, + 1, + &cudnn_data_type)); + + // Set mode-specific attributes + if (attributes.mode == PointwiseMode_t::RELU_FWD || attributes.mode == PointwiseMode_t::RELU_BWD) { + cudnnNanPropagation_t nan_propagation = CUDNN_PROPAGATE_NAN; + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_descriptor.get_raw_desc(), + CUDNN_ATTR_POINTWISE_NAN_PROPAGATION, + CUDNN_TYPE_NAN_PROPOGATION, + 1, + &nan_propagation)); + + double lower_clip = attributes.relu_lower_clip.value_or(0.0); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_descriptor.get_raw_desc(), + CUDNN_ATTR_POINTWISE_RELU_LOWER_CLIP, + CUDNN_TYPE_DOUBLE, + 1, + &lower_clip)); + + double upper_clip = attributes.relu_upper_clip.value_or(std::numeric_limits<double>::max()); + if (attributes.compute_data_type == DataType_t::FLOAT) { + upper_clip = std::min<double>(upper_clip, std::numeric_limits<float>::max()); + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_descriptor.get_raw_desc(), + CUDNN_ATTR_POINTWISE_RELU_UPPER_CLIP, + CUDNN_TYPE_DOUBLE, + 1, + &upper_clip)); + + double lower_clip_slope = attributes.relu_lower_clip_slope.value_or(0.0); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_descriptor.get_raw_desc(), + CUDNN_ATTR_POINTWISE_RELU_LOWER_CLIP_SLOPE, + CUDNN_TYPE_DOUBLE, + 1, + &lower_clip_slope)); + } else if (attributes.mode == PointwiseMode_t::ELU_FWD || attributes.mode == PointwiseMode_t::ELU_BWD) { + double elu_alpha = attributes.elu_alpha.value_or(1.0); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute( + pointwise_descriptor.get_raw_desc(), CUDNN_ATTR_POINTWISE_ELU_ALPHA, CUDNN_TYPE_DOUBLE, 1, &elu_alpha)); + } else if (attributes.mode == PointwiseMode_t::SOFTPLUS_FWD || + attributes.mode == PointwiseMode_t::SOFTPLUS_BWD) { + double softplus_beta = attributes.softplus_beta.value_or(1.0); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_descriptor.get_raw_desc(), + CUDNN_ATTR_POINTWISE_SOFTPLUS_BETA, + CUDNN_TYPE_DOUBLE, + 1, + &softplus_beta)); + } else if (attributes.mode == PointwiseMode_t::SWISH_FWD || attributes.mode == PointwiseMode_t::SWISH_BWD) { + double swish_beta = attributes.swish_beta.value_or(1.0); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_descriptor.get_raw_desc(), + CUDNN_ATTR_POINTWISE_SWISH_BETA, + CUDNN_TYPE_DOUBLE, + 1, + &swish_beta)); + } else if (attributes.mode == PointwiseMode_t::GEN_INDEX) { + int64_t axis = attributes.get_axis().value_or(-1); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute( + pointwise_descriptor.get_raw_desc(), CUDNN_ATTR_POINTWISE_AXIS, CUDNN_TYPE_INT64, 1, &axis)); + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(pointwise_descriptor.get_raw_desc())); + CUDNN_FE_LOG_LABEL_ENDL(pointwise_descriptor); + + // Create operation by directly calling cuDNN backend API + Operation_v8 pointwise_operation; + + _CUDNN_CHECK_CUDNN_ERROR( + pointwise_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR)); + + // Set the pointwise descriptor + auto pw_desc_ptr = pointwise_descriptor.get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_POINTWISE_PW_DESCRIPTOR, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &pw_desc_ptr)); + + auto const port_count = get_pointwise_mode_port_count(attributes.mode); + bool const is_activation_bwd = detail::is_activation_backward_mode(attributes.mode); + + if (is_activation_bwd) { + // Backward mode: IN_0 is dy, IN_1 is x, OUT_0 is dx + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(IN_0, Pointwise_attributes::input_names::IN_0); + auto dy_desc = tensors.at(IN_0->second->get_uid())->get_raw_desc(); + + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(IN_1, Pointwise_attributes::input_names::IN_1); + auto x_desc = tensors.at(IN_1->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_POINTWISE_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &x_desc)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_POINTWISE_DYDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dy_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(OUT_0, Pointwise_attributes::output_names::OUT_0); + auto dx_desc = tensors.at(OUT_0->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_POINTWISE_DXDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dx_desc)); + } else { + // Forward mode + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(IN_0, Pointwise_attributes::input_names::IN_0); + auto x_desc = tensors.at(IN_0->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_POINTWISE_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &x_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(OUT_0, Pointwise_attributes::output_names::OUT_0); + auto y_desc = tensors.at(OUT_0->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_POINTWISE_YDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &y_desc)); + + if (port_count >= 3) { + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(IN_1, Pointwise_attributes::input_names::IN_1); + auto b_desc = tensors.at(IN_1->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_POINTWISE_BDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &b_desc)); + } + + if (port_count >= 4) { + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(IN_2, Pointwise_attributes::input_names::IN_2); + auto t_desc = tensors.at(IN_2->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_POINTWISE_TDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &t_desc)); + } + } + + // Set alpha scaling factors (always set to 1.0) + float alpha1 = 1.0f; + float alpha2 = 1.0f; + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute( + pointwise_operation.get_raw_desc(), CUDNN_ATTR_OPERATION_POINTWISE_ALPHA1, CUDNN_TYPE_FLOAT, 1, &alpha1)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute( + pointwise_operation.get_raw_desc(), CUDNN_ATTR_OPERATION_POINTWISE_ALPHA2, CUDNN_TYPE_FLOAT, 1, &alpha2)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(pointwise_operation.get_raw_desc())); + + operations.push_back(std::make_shared<Operation_v8>(std::move(pointwise_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"({"tag": "POINTWISE"})"_json); + } +#endif +}; + +inline void +INode::pointwise(std::shared_ptr<Tensor_attributes> a, + Pointwise_attributes attributes, + std::shared_ptr<Tensor_attributes> c) { + attributes.inputs[Pointwise_attributes::input_names::IN_0] = a; + attributes.outputs[Pointwise_attributes::output_names::OUT_0] = c; + sub_nodes.emplace_back(std::make_unique<PointwiseNode>(std::move(attributes), context)); +} + +inline void +INode::pointwise(std::shared_ptr<Tensor_attributes> a, + std::shared_ptr<Tensor_attributes> b, + Pointwise_attributes attributes, + std::shared_ptr<Tensor_attributes> c) { + attributes.inputs[Pointwise_attributes::input_names::IN_0] = a; + attributes.inputs[Pointwise_attributes::input_names::IN_1] = b; + attributes.outputs[Pointwise_attributes::output_names::OUT_0] = c; + sub_nodes.emplace_back(std::make_unique<PointwiseNode>(std::move(attributes), context)); +} + +inline void +INode::pointwise(std::shared_ptr<Tensor_attributes> a, + std::shared_ptr<Tensor_attributes> b, + std::shared_ptr<Tensor_attributes> c, + Pointwise_attributes attributes, + std::shared_ptr<Tensor_attributes> d) { + attributes.inputs[Pointwise_attributes::input_names::IN_0] = a; + attributes.inputs[Pointwise_attributes::input_names::IN_1] = b; + attributes.inputs[Pointwise_attributes::input_names::IN_2] = c; + attributes.outputs[Pointwise_attributes::output_names::OUT_0] = d; + sub_nodes.emplace_back(std::make_unique<PointwiseNode>(std::move(attributes), context)); +} + +inline std::shared_ptr<Tensor_attributes> +INode::pointwise(std::shared_ptr<Tensor_attributes> a, Pointwise_attributes attributes) { + if (attributes.name.empty()) { + attributes.name += std::to_string(sub_nodes.size()); + } + attributes.inputs[Pointwise_attributes::input_names::IN_0] = a; + if (a->get_name().empty()) { + a->set_name(attributes.name + "::IN_0"); + }; + auto OUT_0 = attributes.outputs[Pointwise_attributes::output_names::OUT_0] = + output_tensor(attributes.name + "::OUT_0"); + + sub_nodes.emplace_back(std::make_unique<PointwiseNode>(std::move(attributes), context)); + return OUT_0; +} + +inline std::shared_ptr<Tensor_attributes> +INode::pointwise(std::shared_ptr<Tensor_attributes> a, + std::shared_ptr<Tensor_attributes> b, + Pointwise_attributes attributes) { + if (attributes.name.empty()) { + attributes.name += std::to_string(sub_nodes.size()); + } + attributes.inputs[Pointwise_attributes::input_names::IN_0] = a; + attributes.inputs[Pointwise_attributes::input_names::IN_1] = b; + if (a->get_name().empty()) { + a->set_name(attributes.name + "::IN_0"); + }; + if (b->get_name().empty()) { + b->set_name(attributes.name + "::IN_1"); + }; + auto OUT_0 = attributes.outputs[Pointwise_attributes::output_names::OUT_0] = + output_tensor(attributes.name + "::OUT_0"); + + sub_nodes.emplace_back(std::make_unique<PointwiseNode>(std::move(attributes), context)); + return OUT_0; +} + +inline std::shared_ptr<Tensor_attributes> +INode::pointwise(std::shared_ptr<Tensor_attributes> a, + std::shared_ptr<Tensor_attributes> b, + std::shared_ptr<Tensor_attributes> c, + Pointwise_attributes attributes) { + if (attributes.name.empty()) { + attributes.name += std::to_string(sub_nodes.size()); + } + attributes.inputs[Pointwise_attributes::input_names::IN_0] = a; + attributes.inputs[Pointwise_attributes::input_names::IN_1] = b; + attributes.inputs[Pointwise_attributes::input_names::IN_2] = c; + if (a->get_name().empty()) { + a->set_name(attributes.name + "::IN_0"); + }; + if (b->get_name().empty()) { + b->set_name(attributes.name + "::IN_1"); + }; + if (c->get_name().empty()) { + c->set_name(attributes.name + "::IN_2"); + }; + auto OUT_0 = attributes.outputs[Pointwise_attributes::output_names::OUT_0] = + output_tensor(attributes.name + "::OUT_0"); + + sub_nodes.emplace_back(std::make_unique<PointwiseNode>(std::move(attributes), context)); + return OUT_0; +} +} // namespace cudnn_frontend::graph
\ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/reduction.h b/deps/cudnn-frontend/include/cudnn_frontend/node/reduction.h new file mode 100644 index 0000000..5944b69 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/reduction.h @@ -0,0 +1,232 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend::graph { + +class ReductionNode : public NodeCRTP<ReductionNode> { + public: + Reduction_attributes attributes; + + ReductionNode(Reduction_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::REDUCTION; + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating ReductionNode " << attributes.name); + + if (attributes.get_is_deterministic() && detail::get_backend_version() < 91100) { + return {error_code_t::GRAPH_NOT_SUPPORTED, "DETERMINISTIC mode is not supported in cudnn version < 9.11.0"}; + } + return {error_code_t::OK, ""}; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for reduction node " << attributes.name); + + attributes.fill_from_context(context); + + // Only inferrencing from IN_0 to OUT_0 works today. + auto x_tensor = attributes.inputs[Reduction_attributes::input_names::X]; + auto y_tensor = attributes.outputs[Reduction_attributes::output_names::Y]; + + auto const& x_tensor_dim = x_tensor->get_dim(); + auto y_tensor_dim = y_tensor->get_dim(); + // Only infer dims and strides if user did not set them + if (y_tensor_dim.empty()) { + y_tensor->set_dim(x_tensor_dim); + } + if (y_tensor->get_stride().empty()) { + auto const& y_dim = y_tensor->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(y_dim.size()); + y_tensor->set_stride(detail::generate_stride(y_dim, stride_order)); + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations, + std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: " << "Building ReductionNode operations " << attributes.name << " "); + + // Create reduction descriptor by directly calling cuDNN backend API + ReductionDesc_v8 reduction_descriptor; + + // 1. Create the backend descriptor + + _CUDNN_CHECK_CUDNN_ERROR( + reduction_descriptor.initialize_managed_backend_pointer(CUDNN_BACKEND_REDUCTION_DESCRIPTOR)); + + // 2. Set compute type attribute + cudnnDataType_t cudnn_data_type; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.compute_data_type, cudnn_data_type)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(reduction_descriptor.get_raw_desc(), + CUDNN_ATTR_REDUCTION_COMP_TYPE, + CUDNN_TYPE_DATA_TYPE, + 1, + &cudnn_data_type)); + + // 3. Set reduction operator attribute + cudnnReduceTensorOp_t cudnn_reduction_mode; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.get_mode().value(), cudnn_reduction_mode)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(reduction_descriptor.get_raw_desc(), + CUDNN_ATTR_REDUCTION_OPERATOR, + CUDNN_TYPE_REDUCTION_OPERATOR_TYPE, + 1, + &cudnn_reduction_mode)); + + // 4. Set deterministic mode if supported +#if (CUDNN_VERSION >= 91100) + if (detail::get_backend_version() >= 91100) { + bool is_deterministic = attributes.get_is_deterministic(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(reduction_descriptor.get_raw_desc(), + CUDNN_ATTR_REDUCTION_IS_DETERMINISTIC, + CUDNN_TYPE_BOOLEAN, + 1, + &is_deterministic)); + } +#endif + + // 5. Finalize the descriptor + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(reduction_descriptor.get_raw_desc())); + CUDNN_FE_LOG_LABEL_ENDL(reduction_descriptor); + + // Create operation by directly calling cuDNN backend API + Operation_v8 reduction_operation; + + // Validate input tensors are set + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Reduction_attributes::input_names::X); + auto Group_offset = attributes.inputs.find(Reduction_attributes::input_names::Group_offset); + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(Y, Reduction_attributes::output_names::Y); + + // 1. Create the backend operation descriptor + + _CUDNN_CHECK_CUDNN_ERROR( + reduction_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_REDUCTION_DESCRIPTOR)); + + // 2. Set the reduction descriptor attribute + auto reduction_desc_ptr = reduction_descriptor.get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(reduction_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_REDUCTION_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &reduction_desc_ptr)); + + // 3. Set the input tensor (X) descriptor attribute + auto x_backend_desc = tensors.at(X->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(reduction_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_REDUCTION_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &x_backend_desc)); + + // 4. Set the output tensor (Y) descriptor attribute + auto y_backend_desc = tensors.at(Y->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(reduction_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_REDUCTION_YDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &y_backend_desc)); + + // 5. Set the optional group offset tensor descriptor + if (Group_offset != attributes.inputs.end() && Group_offset->second != nullptr) { +#if (CUDNN_VERSION >= 92400) + RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 92400, + error_code_t::GRAPH_NOT_SUPPORTED, + "Reduction group_offset is not supported in cudnn version < 9.24.0"); + auto group_offset_backend_desc = tensors.at(Group_offset->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(reduction_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_REDUCTION_GROUP_OFFSET_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &group_offset_backend_desc)); +#else + return {error_code_t::GRAPH_NOT_SUPPORTED, + "Reduction group_offset is not supported in cudnn version < 9.24.0"}; +#endif + } + + // 6. Finalize the operation descriptor + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(reduction_operation.get_raw_desc())); + + operations.push_back(std::make_shared<Operation_v8>(std::move(reduction_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"({"tag": "REDUCTION"})"_json); + } +#endif +}; + +inline void +INode::reduction(std::shared_ptr<Tensor_attributes> a, + Reduction_attributes attributes, + std::shared_ptr<Tensor_attributes> c) { + attributes.inputs[Reduction_attributes::input_names::X] = a; + attributes.outputs[Reduction_attributes::output_names::Y] = c; + sub_nodes.emplace_back(std::make_unique<ReductionNode>(std::move(attributes), context)); +} + +inline void +INode::reduction(std::shared_ptr<Tensor_attributes> a, + std::shared_ptr<Tensor_attributes> group_offset, + Reduction_attributes attributes, + std::shared_ptr<Tensor_attributes> c) { + attributes.inputs[Reduction_attributes::input_names::X] = a; + attributes.inputs[Reduction_attributes::input_names::Group_offset] = group_offset; + attributes.outputs[Reduction_attributes::output_names::Y] = c; + sub_nodes.emplace_back(std::make_unique<ReductionNode>(std::move(attributes), context)); +} + +inline std::shared_ptr<Tensor_attributes> +INode::reduction(std::shared_ptr<Tensor_attributes> input, Reduction_attributes attributes) { + attributes.inputs[Reduction_attributes::input_names::X] = input; + auto Y = attributes.outputs[Reduction_attributes::output_names::Y] = output_tensor(attributes.name + "::Y"); + + sub_nodes.emplace_back(std::make_unique<ReductionNode>(std::move(attributes), context)); + return Y; +} + +inline std::shared_ptr<Tensor_attributes> +INode::reduction(std::shared_ptr<Tensor_attributes> input, + std::shared_ptr<Tensor_attributes> group_offset, + Reduction_attributes attributes) { + attributes.inputs[Reduction_attributes::input_names::X] = input; + attributes.inputs[Reduction_attributes::input_names::Group_offset] = group_offset; + auto Y = attributes.outputs[Reduction_attributes::output_names::Y] = output_tensor(attributes.name + "::Y"); + + sub_nodes.emplace_back(std::make_unique<ReductionNode>(std::move(attributes), context)); + return Y; +} +} // namespace cudnn_frontend::graph diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/resample.h b/deps/cudnn-frontend/include/cudnn_frontend/node/resample.h new file mode 100644 index 0000000..34e6031 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/resample.h @@ -0,0 +1,291 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend::graph { + +class ResampleNode : public NodeCRTP<ResampleNode> { + public: + Resample_attributes attributes; + + ResampleNode(Resample_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::RESAMPLE; + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: " << "Validating ResampleNode " << attributes.name); + + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.generate_index.has_value() == false, + error_code_t::ATTRIBUTE_NOT_SET, + "generate_index attribute not set"); + + if (attributes.generate_index.value() == true && attributes.resample_mode == ResampleMode_t::MAXPOOL) { + CUDNN_FE_VALIDATE_OUTPUT_TENSOR(Resample_attributes::output_names::Index); + } + + // Make sure that the mode can be lowered to BE + cudnnResampleMode_t dummy; + RETURN_CUDNN_FRONTEND_ERROR_IF( + detail::convert_to_cudnn_type(attributes.resample_mode, dummy) != CUDNN_STATUS_SUCCESS, + error_code_t::ATTRIBUTE_NOT_SET, + "Invalid resample mode."); + + return {error_code_t::OK, ""}; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for resample node " << attributes.name); + + auto y_tensor = attributes.outputs[Resample_attributes::output_names::Y]; + auto x_tensor = attributes.inputs[Resample_attributes::input_names::X]; + + attributes.fill_from_context(context); + + // If user does not set shape and layout of the output tensor, + // Get it from node attributes + if (y_tensor->get_dim().empty()) { + auto const x_dim = x_tensor->get_dim(); + auto y_dim = y_tensor->get_dim(); + y_dim = x_dim; + + // 2 cause first two dimensions are batch and channels + for (auto dim = 2u; dim < x_dim.size(); ++dim) { + auto spatial_dim = dim - 2u; + y_dim[dim] = + 1 + (x_dim[dim] + attributes.pre_padding[spatial_dim].numerator + + attributes.post_padding[spatial_dim].numerator - attributes.window[spatial_dim].numerator) / + attributes.stride[spatial_dim].numerator; + } + + y_tensor->set_dim(y_dim); + } + + // If layout is not set, generate the strides from layout + if (y_tensor->get_stride().empty()) { + auto const& y_dim = y_tensor->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(y_dim.size()); + y_tensor->set_stride(detail::generate_stride(y_dim, stride_order)); + } + + if (attributes.outputs[Resample_attributes::output_names::Index]) { + auto index_tensor = attributes.outputs[Resample_attributes::output_names::Index]; + index_tensor->set_dim(y_tensor->get_dim()); + + // If layout is not set, generate the strides from layout + if (index_tensor->get_stride().empty()) { + auto const& index_dim = index_tensor->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(index_dim.size()); + index_tensor->set_stride(detail::generate_stride(index_dim, stride_order)); + } + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations, + std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: " << "Building ResampleNode operations " << attributes.name << " "); + + auto number_of_spatial_dim = static_cast<int64_t>(attributes.window.size()); + + // Create resample descriptor by directly calling cuDNN backend API + ResampleDesc_v8 resample_descriptor; + + _CUDNN_CHECK_CUDNN_ERROR( + resample_descriptor.initialize_managed_backend_pointer(CUDNN_BACKEND_RESAMPLE_DESCRIPTOR)); + + // Set resample mode + cudnnResampleMode_t cudnn_resample_mode; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.resample_mode, cudnn_resample_mode)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_descriptor.get_raw_desc(), + CUDNN_ATTR_RESAMPLE_MODE, + CUDNN_TYPE_RESAMPLE_MODE, + 1, + &cudnn_resample_mode)); + + // Set compute type + cudnnDataType_t cudnn_data_type; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.compute_data_type, cudnn_data_type)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_descriptor.get_raw_desc(), + CUDNN_ATTR_RESAMPLE_COMP_TYPE, + CUDNN_TYPE_DATA_TYPE, + 1, + &cudnn_data_type)); + + // Set nan propagation + cudnnNanPropagation_t nan_opt = CUDNN_PROPAGATE_NAN; + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_descriptor.get_raw_desc(), + CUDNN_ATTR_RESAMPLE_NAN_PROPAGATION, + CUDNN_TYPE_NAN_PROPOGATION, + 1, + &nan_opt)); + + // Set padding mode + cudnnPaddingMode_t cudnn_padding_mode; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.padding_mode, cudnn_padding_mode)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_descriptor.get_raw_desc(), + CUDNN_ATTR_RESAMPLE_PADDING_MODE, + CUDNN_TYPE_PADDING_MODE, + 1, + &cudnn_padding_mode)); + + // Set spatial dimensions + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_descriptor.get_raw_desc(), + CUDNN_ATTR_RESAMPLE_SPATIAL_DIMS, + CUDNN_TYPE_INT64, + 1, + &number_of_spatial_dim)); + + // Set window dimensions + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_descriptor.get_raw_desc(), + CUDNN_ATTR_RESAMPLE_WINDOW_DIMS, + CUDNN_TYPE_FRACTION, + number_of_spatial_dim, + attributes.window.data())); + + // Set pre padding + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_descriptor.get_raw_desc(), + CUDNN_ATTR_RESAMPLE_PRE_PADDINGS, + CUDNN_TYPE_FRACTION, + number_of_spatial_dim, + attributes.pre_padding.data())); + + // Set post padding + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_descriptor.get_raw_desc(), + CUDNN_ATTR_RESAMPLE_POST_PADDINGS, + CUDNN_TYPE_FRACTION, + number_of_spatial_dim, + attributes.post_padding.data())); + + // Set strides + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_descriptor.get_raw_desc(), + CUDNN_ATTR_RESAMPLE_STRIDES, + CUDNN_TYPE_FRACTION, + number_of_spatial_dim, + attributes.stride.data())); + + // Finalize the descriptor + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(resample_descriptor.get_raw_desc())); + CUDNN_FE_LOG_LABEL_ENDL(resample_descriptor); + + // Create operation by directly calling cuDNN backend API + Operation_v8 resample_operation; + + _CUDNN_CHECK_CUDNN_ERROR( + resample_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_RESAMPLE_FWD_DESCRIPTOR)); + + // Set input tensor X + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Resample_attributes::input_names::X); + auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_RESAMPLE_FWD_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &x_desc)); + + // Set output tensor Y + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(Y, Resample_attributes::output_names::Y); + auto y_desc = tensors.at(Y->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_RESAMPLE_FWD_YDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &y_desc)); + + // Set alpha and beta + double alpha = 1.0; + double beta = 0.0; + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute( + resample_operation.get_raw_desc(), CUDNN_ATTR_OPERATION_RESAMPLE_FWD_ALPHA, CUDNN_TYPE_DOUBLE, 1, &alpha)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute( + resample_operation.get_raw_desc(), CUDNN_ATTR_OPERATION_RESAMPLE_FWD_BETA, CUDNN_TYPE_DOUBLE, 1, &beta)); + + // Set resample descriptor + auto resample_raw_desc = resample_descriptor.get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_RESAMPLE_FWD_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &resample_raw_desc)); + + // Set index tensor if available + auto index = attributes.outputs.find(Resample_attributes::output_names::Index); + if ((index != attributes.outputs.end()) && (index->second != nullptr)) { + auto idx_desc = tensors.at(index->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_RESAMPLE_FWD_IDXDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &idx_desc)); + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(resample_operation.get_raw_desc())); + + operations.push_back(std::make_shared<Operation_v8>(std::move(resample_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "RESAMPLE"})"_json); + } +#endif +}; + +inline std::array<std::shared_ptr<Tensor_attributes>, 2> +INode::resample(std::shared_ptr<Tensor_attributes> input, Resample_attributes attributes) { + if (attributes.name.empty()) { + attributes.name += std::to_string(sub_nodes.size()); + } + attributes.inputs[Resample_attributes::input_names::X] = input; + auto Y = attributes.outputs[Resample_attributes::output_names::Y] = output_tensor(attributes.name + "::Y"); + std::shared_ptr<Tensor_attributes> Index = nullptr; + if (attributes.generate_index.has_value() && attributes.generate_index.value() == true && + attributes.resample_mode == ResampleMode_t::MAXPOOL) { + Index = attributes.outputs[Resample_attributes::output_names::Index] = + output_tensor(attributes.name + "::Index"); + } + + sub_nodes.emplace_back(std::make_unique<ResampleNode>(std::move(attributes), context)); + return {Y, Index}; +} + +} // namespace cudnn_frontend::graph
\ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/reshape.h b/deps/cudnn-frontend/include/cudnn_frontend/node/reshape.h new file mode 100644 index 0000000..e2b5546 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/reshape.h @@ -0,0 +1,156 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend::graph { + +class ReshapeNode : public NodeCRTP<ReshapeNode> { + public: + Reshape_attributes attributes; + + ReshapeNode(Reshape_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::RESHAPE; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for reshape node " << attributes.name); + + auto y_tensor = attributes.outputs[Reshape_attributes::output_names::Y]; + + attributes.fill_from_context(context); + + // If user does not set shape and layout of the output tensor, + // Get it from node attributes + // If layout is not set, generate the strides from layout + + if (y_tensor->get_dim().empty() && attributes.get_dim().size()) { + y_tensor->set_dim(attributes.dim); + } + + // Graph producers may represent scalar reshape outputs + // as rank-0 tensors (dim={}). Without promotion, stride inference below + // would call generate_NHWC_stride_order(0), which is UB, and the cuDNN + // backend rejects rank-0 descriptors regardless. Promote to canonical + // rank-1 length-1 -- value-preserving because a scalar has volume 1 -- + // while keeping the normalization node-local so downstream broadcast/shape + // inference remains intact. + if (y_tensor->get_dim().empty()) { + y_tensor->set_dim({1}); + if (y_tensor->get_stride().empty()) { + y_tensor->set_stride({1}); + } + } + + if (y_tensor->get_stride().empty()) { + if (attributes.get_stride().size()) { + y_tensor->set_stride(attributes.get_stride()); + } else { + auto const& y_dim = y_tensor->get_dim(); + // Default to NHWC for multi-axis tensors. generate_NHWC_stride_order + // indexes stride_order[1] and assumes num_dims >= 2; use row-major + // for scalars (0-D) and vectors (1-D). + const int64_t rank = static_cast<int64_t>(y_dim.size()); + std::vector<int64_t> const stride_order = + rank < 2 ? detail::generate_row_major_stride_order(rank) : detail::generate_NHWC_stride_order(rank); + y_tensor->set_stride(detail::generate_stride(y_dim, stride_order)); + } + } + + if (y_tensor->get_dim().empty() || y_tensor->get_stride().empty()) { + return {error_code_t::SHAPE_DEDUCTION_FAILED, "Reshape node output shape deduction failed"}; + } + + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Reshape_attributes::input_names::X); + auto const& input_data_type = X->second->get_data_type(); + if (y_tensor->get_data_type() == DataType_t::NOT_SET) { + y_tensor->set_data_type(input_data_type); + } else if (attributes.get_reshape_mode() == ReshapeMode_t::LOGICAL) { + // Lexicographic reshape preserves element type; reject inconsistent metadata. + // VIEW_ONLY paths (e.g. SDPA backward) may set Y dtype after reshape to match a consumer. + RETURN_CUDNN_FRONTEND_ERROR_IF( + y_tensor->get_data_type() != input_data_type, + error_code_t::INVALID_VALUE, + "Output and input tensor data types must match for LOGICAL reshape operation."); + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations, + std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: " << "Building ReshapeNode operations " << attributes.name << " "); + + // Create operation by directly calling cuDNN backend API + Operation_v8 reshape_operation; + + _CUDNN_CHECK_CUDNN_ERROR( + reshape_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_RESHAPE_DESCRIPTOR)); + + // Set input tensor X + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Reshape_attributes::input_names::X); + auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(reshape_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_RESHAPE_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &x_desc)); +#if (CUDNN_VERSION >= 92200) + // Set reshape mode + cudnnBackendReshapeMode_t cudnn_reshape_mode; + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.get_reshape_mode(), cudnn_reshape_mode)); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(reshape_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_RESHAPE_MODE, + CUDNN_TYPE_RESHAPE_MODE, + 1, + &cudnn_reshape_mode)); +#endif + // Set output tensor Y + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(Y, Reshape_attributes::output_names::Y); + auto y_desc = tensors.at(Y->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(reshape_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_RESHAPE_YDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &y_desc)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(reshape_operation.get_raw_desc())); + + operations.push_back(std::make_shared<Operation_v8>(std::move(reshape_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "RESHAPE"})"_json); + } +#endif +}; + +inline std::shared_ptr<Tensor_attributes> +INode::reshape(std::shared_ptr<Tensor_attributes> input, Reshape_attributes attributes) { + attributes.inputs[Reshape_attributes::input_names::X] = input; + auto Y = attributes.outputs[Reshape_attributes::output_names::Y] = output_tensor(attributes.name + "::Y"); + + sub_nodes.emplace_back(std::make_unique<ReshapeNode>(std::move(attributes), context)); + return Y; +} + +} // namespace cudnn_frontend::graph
\ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/rmsnorm.h b/deps/cudnn-frontend/include/cudnn_frontend/node/rmsnorm.h new file mode 100644 index 0000000..bc1f37d --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/rmsnorm.h @@ -0,0 +1,406 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend { + +namespace graph { +class RMSNormNode : public NodeCRTP<RMSNormNode> { + public: + Rmsnorm_attributes attributes; + + RMSNormNode(Rmsnorm_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::RMSNORM; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for rmsnorm node " << attributes.name); + + attributes.fill_from_context(context); + + auto X = attributes.inputs[Rmsnorm_attributes::input_names::X]; + auto Y = attributes.outputs[Rmsnorm_attributes::output_names::Y]; + + // Only infer dims and strides if user did not set them + if (Y->get_dim().empty()) { + Y->set_dim(X->get_dim()); + } + if (Y->get_stride().empty()) { + Y->set_stride(X->get_stride()); + } + + if (attributes.forward_phase == NormFwdPhase_t::TRAINING) { + auto inv_var = attributes.outputs[Rmsnorm_attributes::output_names::INV_VARIANCE]; + // Only infer dims and strides if user did not set them + if (inv_var->get_dim().empty()) { + auto inv_var_dim = X->get_dim(); + auto scale = attributes.inputs[Rmsnorm_attributes::input_names::SCALE]; + if (scale->get_dim().empty()) { + // mean inv_var dim is n,1,1,1 + for (size_t i = 1; i < inv_var_dim.size(); i++) { + inv_var_dim[i] = 1; + } + } else { + for (size_t i = 0; i < inv_var_dim.size(); i++) { + if (scale->get_dim()[i] != 1) { + inv_var_dim[i] = 1; + } + } + } + inv_var->set_dim(inv_var_dim); + } + if (inv_var->get_stride().empty()) { + auto const& inv_var_dim = inv_var->get_dim(); + std::vector<int64_t> stride_order; + CHECK_CUDNN_FRONTEND_ERROR( + detail::generate_stride_order_preserving_format(X->get_stride(), inv_var_dim.size(), stride_order)); + inv_var->set_stride(detail::generate_stride(inv_var_dim, stride_order)); + } + } + return {error_code_t::OK, ""}; + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating RMSNormNode " << attributes.name); + + // Norm forward phase should be set + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.forward_phase == NormFwdPhase_t::NOT_SET, + error_code_t::ATTRIBUTE_NOT_SET, + "Forward phase not set of rmsnorm node."); + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations, + std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: Building RMSNormNode operations " << attributes.name << " "); + + // Create operation by directly calling cuDNN backend API + Operation_v8 rmsnorm_operation; + + _CUDNN_CHECK_CUDNN_ERROR( + rmsnorm_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_NORM_FORWARD_DESCRIPTOR)); + + // Set norm mode to RMS_NORM + cudnnBackendNormMode_t cudnn_norm_mode; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormMode_t::RMS_NORM, cudnn_norm_mode)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rmsnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_MODE, + CUDNN_TYPE_NORM_MODE, + 1, + &cudnn_norm_mode)); + + // Set forward phase + cudnnBackendNormFwdPhase_t cudnn_norm_fwd_phase; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.forward_phase, cudnn_norm_fwd_phase)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rmsnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_PHASE, + CUDNN_TYPE_NORM_FWD_PHASE, + 1, + &cudnn_norm_fwd_phase)); + + // Set input tensor X + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Rmsnorm_attributes::input_names::X); + auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rmsnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &x_desc)); + + // Set scale tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SCALE, Rmsnorm_attributes::input_names::SCALE); + auto scale_desc = tensors.at(SCALE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rmsnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_SCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &scale_desc)); + + // Set epsilon tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(EPSILON, Rmsnorm_attributes::input_names::EPSILON); + auto epsilon_desc = tensors.at(EPSILON->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rmsnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_EPSILON_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &epsilon_desc)); + + // Set output tensor Y + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(Y, Rmsnorm_attributes::output_names::Y); + auto y_desc = tensors.at(Y->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rmsnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_YDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &y_desc)); + + // Set inv_variance for training phase + if (attributes.forward_phase == NormFwdPhase_t::TRAINING) { + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(INV_VARIANCE, Rmsnorm_attributes::output_names::INV_VARIANCE); + auto inv_var_desc = tensors.at(INV_VARIANCE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rmsnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_INV_VARIANCE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &inv_var_desc)); + } + + // Set optional bias tensor + auto BIAS = attributes.inputs.find(Rmsnorm_attributes::input_names::BIAS); + if ((BIAS != attributes.inputs.end()) && (BIAS->second != nullptr)) { + auto bias_desc = tensors.at(BIAS->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rmsnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_BIAS_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &bias_desc)); + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(rmsnorm_operation.get_raw_desc())); + + operations.push_back(std::make_shared<Operation_v8>(std::move(rmsnorm_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "RMS_NORM"})"_json); + } +#endif +}; + +class DRMSNormNode : public NodeCRTP<DRMSNormNode> { + public: + Rmsnorm_backward_attributes attributes; + + DRMSNormNode(Rmsnorm_backward_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::DRMSNorm; + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating DRMSNormNode node " << attributes.name); + + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.use_dbias.has_value() == false, + error_code_t::ATTRIBUTE_NOT_SET, + "DRMSNormNode node needs has_bias(bool) to be called."); + + return {error_code_t::OK, ""}; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for DRMSNorm node " << attributes.name); + + attributes.fill_from_context(context); + + // TODO: Only inferencing from X works today. + auto X = attributes.inputs[Rmsnorm_backward_attributes::input_names::X]; + auto const x_tensor_dim = X->get_dim(); + + auto DY = attributes.inputs[Rmsnorm_backward_attributes::input_names::DY]; + auto dy_tensor_dim = DY->get_dim(); + + // Only infer dims and strides if user did not set them + if (dy_tensor_dim.empty()) { + dy_tensor_dim.resize(x_tensor_dim.size()); + DY->set_dim(x_tensor_dim); + } + if (DY->get_stride().empty()) { + auto const& DY_dim = DY->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(DY_dim.size()); + DY->set_stride(detail::generate_stride(DY_dim, stride_order)); + } + + auto DX = attributes.outputs[Rmsnorm_backward_attributes::output_names::DX]; + auto dx_tensor_dim = DX->get_dim(); + // Only infer dims and strides if user did not set them + if (dx_tensor_dim.empty()) { + dx_tensor_dim.resize(x_tensor_dim.size()); + DX->set_dim(x_tensor_dim); + } + if (DX->get_stride().empty()) { + auto const& DX_dim = DX->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(DX_dim.size()); + DX->set_stride(detail::generate_stride(DX_dim, stride_order)); + } + + auto scale_bias_dim = X->get_dim(); + scale_bias_dim[0] = 1; + + // Set channel length tensors + auto infer_scale_bias_tensors = [&scale_bias_dim](std::shared_ptr<Tensor_attributes>& T) { + auto tensor_dim = T->get_dim(); + // Only infer dims and strides if user did not set them + if (tensor_dim.empty()) { + T->set_dim(scale_bias_dim); + } + if (T->get_stride().empty()) { + auto const& T_dim = T->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(T_dim.size()); + T->set_stride(detail::generate_stride(T_dim, stride_order)); + } + }; + + infer_scale_bias_tensors(attributes.outputs[Rmsnorm_backward_attributes::output_names::DSCALE]); + if (attributes.use_dbias.value()) { + infer_scale_bias_tensors(attributes.outputs[Rmsnorm_backward_attributes::output_names::DBIAS]); + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations, + std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: Building DRMSNormNode operations " << attributes.name << " "); + + // Create operation by directly calling cuDNN backend API + Operation_v8 drmsnorm_operation; + + _CUDNN_CHECK_CUDNN_ERROR( + drmsnorm_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_NORM_BACKWARD_DESCRIPTOR)); + + // Set norm mode to RMS_NORM + cudnnBackendNormMode_t cudnn_norm_mode; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormMode_t::RMS_NORM, cudnn_norm_mode)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(drmsnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_MODE, + CUDNN_TYPE_NORM_MODE, + 1, + &cudnn_norm_mode)); + + // Set input tensor X + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Rmsnorm_backward_attributes::input_names::X); + auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(drmsnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &x_desc)); + + // Set DY tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(DY, Rmsnorm_backward_attributes::input_names::DY); + auto dy_desc = tensors.at(DY->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(drmsnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_DYDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dy_desc)); + + // Set scale tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SCALE, Rmsnorm_backward_attributes::input_names::SCALE); + auto scale_desc = tensors.at(SCALE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(drmsnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_SCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &scale_desc)); + + // Set inv_variance tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(INV_VARIANCE, Rmsnorm_backward_attributes::input_names::INV_VARIANCE); + auto inv_var_desc = tensors.at(INV_VARIANCE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(drmsnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_INV_VARIANCE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &inv_var_desc)); + + // Set DSCALE output tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DSCALE, Rmsnorm_backward_attributes::output_names::DSCALE); + auto dscale_desc = tensors.at(DSCALE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(drmsnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_DSCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dscale_desc)); + + // Set optional DBIAS output tensor + if (attributes.use_dbias.value()) { + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DBIAS, Rmsnorm_backward_attributes::output_names::DBIAS); + auto dbias_desc = tensors.at(DBIAS->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(drmsnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_DBIAS_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dbias_desc)); + } + + // Set DX output tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DX, Rmsnorm_backward_attributes::output_names::DX); + auto dx_desc = tensors.at(DX->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(drmsnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_DXDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dx_desc)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(drmsnorm_operation.get_raw_desc())); + + operations.push_back(std::make_shared<Operation_v8>(std::move(drmsnorm_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "RMS_NORM_BPROP"})"_json); + } +#endif +}; + +} // namespace graph + +} // namespace cudnn_frontend
\ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/rng.h b/deps/cudnn-frontend/include/cudnn_frontend/node/rng.h new file mode 100644 index 0000000..09b762d --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/rng.h @@ -0,0 +1,187 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend::graph { + +class RngNode : public NodeCRTP<RngNode> { + public: + Rng_attributes attributes; + + RngNode(Rng_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::RNG; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for rng node " << attributes.name); + + auto y_tensor = attributes.outputs[Rng_attributes::output_names::Y]; + + attributes.fill_from_context(context); + + // If user does not set shape and layout of the generated tensor, + // Get it from node attributes + // If layout is not set, generate the strides from layout + + if (y_tensor->get_dim().empty() && attributes.get_dim().size()) { + y_tensor->set_dim(attributes.dim); + } + + if (y_tensor->get_stride().empty()) { + if (attributes.get_stride().size()) { + y_tensor->set_stride(attributes.get_stride()); + } else { + auto const& y_dim = y_tensor->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(y_dim.size()); + y_tensor->set_stride(detail::generate_stride(y_dim, stride_order)); + } + } + + if (y_tensor->get_dim().empty() || y_tensor->get_stride().empty()) { + return {error_code_t::SHAPE_DEDUCTION_FAILED, "RNG node output shape deduction failed"}; + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations, + std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: " << "Building RngNode operations " << attributes.name << " "); + + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.get_distribution() != RngDistribution_t::BERNOULLI, + error_code_t::ATTRIBUTE_NOT_SET, + "no other distribution except bernoulli supported."); + + // Create RNG descriptor by directly calling cuDNN backend API + RngDesc_v8 rng_descriptor; + + _CUDNN_CHECK_CUDNN_ERROR(rng_descriptor.initialize_managed_backend_pointer(CUDNN_BACKEND_RNG_DESCRIPTOR)); + + // Set distribution type + cudnnRngDistribution_t cudnn_rng_distribution; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.get_distribution(), cudnn_rng_distribution)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rng_descriptor.get_raw_desc(), + CUDNN_ATTR_RNG_DISTRIBUTION, + CUDNN_TYPE_RNG_DISTRIBUTION, + 1, + &cudnn_rng_distribution)); + + // Set Bernoulli distribution probability + double bernoulli_prob = attributes.get_bernoulli_probability().value(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rng_descriptor.get_raw_desc(), + CUDNN_ATTR_RNG_BERNOULLI_DIST_PROBABILITY, + CUDNN_TYPE_DOUBLE, + 1, + &bernoulli_prob)); + + // Finalize the descriptor + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(rng_descriptor.get_raw_desc())); + CUDNN_FE_LOG_LABEL_ENDL(rng_descriptor); + + // Create operation by directly calling cuDNN backend API + Operation_v8 rng_operation; + + _CUDNN_CHECK_CUDNN_ERROR( + rng_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_RNG_DESCRIPTOR)); + + // Set output tensor Y + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(Y, Rng_attributes::output_names::Y); + auto y_desc = tensors.at(Y->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute( + rng_operation.get_raw_desc(), CUDNN_ATTR_OPERATION_RNG_YDESC, CUDNN_TYPE_BACKEND_DESCRIPTOR, 1, &y_desc)); + + // Set RNG descriptor + auto rng_raw_desc = rng_descriptor.get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rng_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_RNG_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &rng_raw_desc)); + + if (attributes.seed.has_value()) { + // Set seed as int64_t value + int64_t seed_value = attributes.get_seed().value(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute( + rng_operation.get_raw_desc(), CUDNN_ATTR_OPERATION_RNG_SEED, CUDNN_TYPE_INT64, 1, &seed_value)); + } else { + // Set seed tensor descriptor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(Seed, Rng_attributes::input_names::Seed); + auto seed_desc = tensors.at(Seed->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rng_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_RNG_SEED, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &seed_desc)); + + // Set offset tensor descriptor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(Offset, Rng_attributes::input_names::Offset); + auto offset_desc = tensors.at(Offset->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rng_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_RNG_OFFSET_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &offset_desc)); + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(rng_operation.get_raw_desc())); + + operations.push_back(std::make_shared<Operation_v8>(std::move(rng_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "RNG"})"_json); + } +#endif +}; + +inline void +INode::rng(std::shared_ptr<Tensor_attributes> seed, + std::shared_ptr<Tensor_attributes> offset, + Rng_attributes attributes, + std::shared_ptr<Tensor_attributes> y) { + attributes.inputs[Rng_attributes::input_names::Seed] = seed; + attributes.inputs[Rng_attributes::input_names::Offset] = offset; + attributes.outputs[Rng_attributes::output_names::Y] = y; + sub_nodes.emplace_back(std::make_unique<RngNode>(std::move(attributes), context)); +} + +inline std::shared_ptr<Tensor_attributes> +INode::rng(std::shared_ptr<Tensor_attributes> seed, + std::shared_ptr<Tensor_attributes> offset, + Rng_attributes attributes) { + attributes.inputs[Rng_attributes::input_names::Seed] = seed; + attributes.inputs[Rng_attributes::input_names::Offset] = offset; + auto Y = attributes.outputs[Rng_attributes::output_names::Y] = output_tensor(attributes.name + "::Y"); + + sub_nodes.emplace_back(std::make_unique<RngNode>(std::move(attributes), context)); + return Y; +} +} // namespace cudnn_frontend::graph
\ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/rope.h b/deps/cudnn-frontend/include/cudnn_frontend/node/rope.h new file mode 100644 index 0000000..01b05c4 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/rope.h @@ -0,0 +1,146 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend { + +namespace graph { +class RoPENode : public NodeCRTP<RoPENode> { + public: + RoPE_attributes attributes; + + RoPENode(RoPE_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::ROPE; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for RoPE node " << attributes.name); + + attributes.fill_from_context(context); + + auto INPUT = attributes.inputs[RoPE_attributes::input_names::INPUT]; + auto OUTPUT = attributes.outputs[RoPE_attributes::output_names::OUTPUT]; + + if (OUTPUT->get_dim().empty()) { + OUTPUT->set_dim(INPUT->get_dim()); + } + if (OUTPUT->get_stride().empty()) { + OUTPUT->set_stride(INPUT->get_stride()); + } + + return {error_code_t::OK, ""}; + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating RoPENode " << attributes.name); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.inputs.find(RoPE_attributes::input_names::INPUT) == attributes.inputs.end(), + error_code_t::ATTRIBUTE_NOT_SET, + "RoPE node requires INPUT tensor."); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.inputs.find(RoPE_attributes::input_names::FREQS) == attributes.inputs.end(), + error_code_t::ATTRIBUTE_NOT_SET, + "RoPE node requires FREQS tensor."); + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations, + std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(operations); + CUDNN_FE_LOG_LABEL("INFO: Building RoPENode operations " << attributes.name << " "); + +#if (CUDNN_VERSION >= 92400) + // Compile- and run-time checks: RoPE op was added in cuDNN 9.24. + RETURN_CUDNN_FRONTEND_ERROR_IF( + detail::get_backend_version() < 92400, + error_code_t::GRAPH_NOT_SUPPORTED, + "CUDNN_BACKEND_OPERATION_ROPE_FWD_DESCRIPTOR is only available starting cuDNN 9.24."); + + auto rope_operation = + make_shared_backend_pointer((cudnnBackendDescriptorType_t)CUDNN_BACKEND_OPERATION_ROPE_FWD_DESCRIPTOR); + + // Set input X + auto X = attributes.inputs.find(RoPE_attributes::input_names::INPUT)->second; + auto backend_x = tensors[X->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rope_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_ROPE_FWD_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_x)); + + // Set freqs + auto FREQS = attributes.inputs.find(RoPE_attributes::input_names::FREQS)->second; + auto backend_freqs = tensors[FREQS->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rope_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_ROPE_FWD_FREQSDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_freqs)); + + // Set output Y + auto Y = attributes.outputs.find(RoPE_attributes::output_names::OUTPUT)->second; + auto backend_y = tensors[Y->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rope_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_ROPE_FWD_YDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_y)); + + // Set output_scale (host fp32 scalar; default 1.0) + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rope_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_ROPE_FWD_OUTPUT_SCALE, + CUDNN_TYPE_FLOAT, + 1, + &attributes.output_scale)); + + // Set rope_dim (int64; 0 = full rotation = input's head_dim) + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rope_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_ROPE_FWD_ROPE_DIM, + CUDNN_TYPE_INT64, + 1, + &attributes.rope_dim)); + + // Finalize + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(rope_operation->get_backend_descriptor())); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + + raw_operations.push_back(rope_operation); + + return {error_code_t::OK, ""}; +#else + CUDNN_FRONTEND_UNUSED(uids_involved_in_operations); + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FRONTEND_UNUSED(tensors); + return {error_code_t::GRAPH_NOT_SUPPORTED, + "CUDNN_BACKEND_OPERATION_ROPE_FWD_DESCRIPTOR is only available starting cuDNN 9.24."}; +#endif + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"({"tag": "ROPE"})"_json); + } +#endif +}; + +} // namespace graph + +} // namespace cudnn_frontend diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/rope_backward.h b/deps/cudnn-frontend/include/cudnn_frontend/node/rope_backward.h new file mode 100644 index 0000000..fd7bf4f --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/rope_backward.h @@ -0,0 +1,143 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend { + +namespace graph { +class RoPEBackwardNode : public NodeCRTP<RoPEBackwardNode> { + public: + RoPE_backward_attributes attributes; + + RoPEBackwardNode(RoPE_backward_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::ROPE_BWD; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for RoPE backward node " << attributes.name); + + attributes.fill_from_context(context); + + auto DY = attributes.inputs[RoPE_backward_attributes::input_names::DY]; + auto DX = attributes.outputs[RoPE_backward_attributes::output_names::DX]; + + if (DX->get_dim().empty()) { + DX->set_dim(DY->get_dim()); + } + if (DX->get_stride().empty()) { + DX->set_stride(DY->get_stride()); + } + + return {error_code_t::OK, ""}; + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating RoPEBackwardNode " << attributes.name); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.inputs.find(RoPE_backward_attributes::input_names::DY) == attributes.inputs.end(), + error_code_t::ATTRIBUTE_NOT_SET, + "RoPE backward node requires DY (input gradient) tensor."); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.inputs.find(RoPE_backward_attributes::input_names::FREQS) == attributes.inputs.end(), + error_code_t::ATTRIBUTE_NOT_SET, + "RoPE backward node requires FREQS tensor."); + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations, + std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(operations); + CUDNN_FE_LOG_LABEL("INFO: Building RoPEBackwardNode operations " << attributes.name << " "); + +#if (CUDNN_VERSION >= 92400) + // Compile- and run-time checks: RoPE bwd op was added in cuDNN 9.24. + RETURN_CUDNN_FRONTEND_ERROR_IF( + detail::get_backend_version() < 92400, + error_code_t::GRAPH_NOT_SUPPORTED, + "CUDNN_BACKEND_OPERATION_ROPE_BWD_DESCRIPTOR is only available starting cuDNN 9.24."); + + auto rope_bwd_operation = + make_shared_backend_pointer((cudnnBackendDescriptorType_t)CUDNN_BACKEND_OPERATION_ROPE_BWD_DESCRIPTOR); + + // Set DY (input gradient) + auto DY = attributes.inputs.find(RoPE_backward_attributes::input_names::DY)->second; + auto backend_dy = tensors[DY->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rope_bwd_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_ROPE_BWD_DYDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_dy)); + + // Set freqs + auto FREQS = attributes.inputs.find(RoPE_backward_attributes::input_names::FREQS)->second; + auto backend_freqs = tensors[FREQS->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rope_bwd_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_ROPE_BWD_FREQSDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_freqs)); + + // Set DX (output gradient) + auto DX = attributes.outputs.find(RoPE_backward_attributes::output_names::DX)->second; + auto backend_dx = tensors[DX->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rope_bwd_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_ROPE_BWD_DXDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_dx)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rope_bwd_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_ROPE_BWD_OUTPUT_SCALE, + CUDNN_TYPE_FLOAT, + 1, + &attributes.output_scale)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rope_bwd_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_ROPE_BWD_ROPE_DIM, + CUDNN_TYPE_INT64, + 1, + &attributes.rope_dim)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(rope_bwd_operation->get_backend_descriptor())); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + + raw_operations.push_back(rope_bwd_operation); + + return {error_code_t::OK, ""}; +#else + CUDNN_FRONTEND_UNUSED(uids_involved_in_operations); + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FRONTEND_UNUSED(tensors); + return {error_code_t::GRAPH_NOT_SUPPORTED, + "CUDNN_BACKEND_OPERATION_ROPE_BWD_DESCRIPTOR is only available starting cuDNN 9.24."}; +#endif + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"({"tag": "ROPE_BWD"})"_json); + } +#endif +}; + +} // namespace graph + +} // namespace cudnn_frontend diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/scaled_dot_product_flash_attention.h b/deps/cudnn-frontend/include/cudnn_frontend/node/scaled_dot_product_flash_attention.h new file mode 100644 index 0000000..c3c37c5 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/scaled_dot_product_flash_attention.h @@ -0,0 +1,2730 @@ + +#pragma once + +#include <cstdlib> +#include <unordered_set> + +#include "../../cudnn_frontend_Heuristics.h" +#include "../../cudnn_frontend_Logging.h" + +#include "../graph_helpers.h" +#include "../node_interface.h" + +#include "diagonal_band_mask.h" +#include "matmul.h" +#include "pointwise.h" +#include "reduction.h" +#include "rng.h" +#include "softmax.h" +#include "paged_cache_load.h" +#include "block_scale_dequantize.h" +#include "sdpa_support_surface.h" + +namespace cudnn_frontend::graph { + +// If a 1-D length/offset index tensor [n] is supplied, promote it to the 4-D +// [n, 1, 1, 1] form that the cuDNN backend requires for seq_len / cu_seq_len and +// ragged-offset tensors. No-op if the tensor is unset or is not 1-D. +inline void +promote_1d_index_tensor_to_4d(std::shared_ptr<Tensor_attributes> const& tensor) { + if (tensor == nullptr) { + return; + } + auto const& dim = tensor->get_dim(); + auto const& stride = tensor->get_stride(); + if (dim.size() == 1) { + int64_t const n = dim[0]; + int64_t const leading_stride = (stride.size() == 1) ? stride[0] : 1; + tensor->set_dim({n, 1, 1, 1}).set_stride({leading_stride, 1, 1, 1}); + } +} + +namespace attn::score_modifiers { + +// clang-format off +inline float get_negative_inf_value(); + +inline std::shared_ptr<Tensor_attributes> causal_mask( + std::shared_ptr<Graph> graph, + std::shared_ptr<Tensor_attributes> attention_score +); + +inline std::shared_ptr<Tensor_attributes> bias( + std::shared_ptr<Graph> graph, + std::shared_ptr<Tensor_attributes> attention_score, + std::shared_ptr<Tensor_attributes> bias_tensor +); + +inline std::shared_ptr<Tensor_attributes> causal_mask_bottom_right( + std::shared_ptr<Graph> graph, + std::shared_ptr<Tensor_attributes> attention_score, + std::shared_ptr<Tensor_attributes> seq_len_q, + std::shared_ptr<Tensor_attributes> seq_len_kv +); + +inline std::shared_ptr<Tensor_attributes> padding_mask( + std::shared_ptr<Graph> graph, + std::shared_ptr<Tensor_attributes> attention_score, + std::shared_ptr<Tensor_attributes> seq_len_kv, + std::shared_ptr<Tensor_attributes> seq_len_q +); + +inline std::shared_ptr<Tensor_attributes> sliding_window_mask( + std::shared_ptr<Graph> graph, + std::shared_ptr<Tensor_attributes> attention_score, + DiagonalAlignment_t diagonal_alignment, + std::optional<int64_t> left_window, + std::optional<int64_t> right_window, + int64_t s_q, + int64_t s_kv, + std::shared_ptr<Tensor_attributes> s_q_ptr, + std::shared_ptr<Tensor_attributes> s_kv_ptr, + std::shared_ptr<Tensor_attributes> cu_s_q_ptr = nullptr, + std::shared_ptr<Tensor_attributes> cu_s_kv_ptr = nullptr +); + +inline std::shared_ptr<Tensor_attributes> alibi_mask( + std::shared_ptr<Graph> graph, + std::shared_ptr<Tensor_attributes> attention_score, + std::shared_ptr<Tensor_attributes>& alibi_slopes, + int64_t h_q, + int64_t& alibi_slopes_size +); + +inline error_t build_operation_subgraph(std::shared_ptr<Graph> graph); +// clang-format on + +} // namespace attn::score_modifiers + +template <typename DerivedT> +class SDPANodeBase : public NodeCRTP<DerivedT> { + protected: + using input_names = SDPA_attributes::input_names; + using output_names = SDPA_attributes::output_names; + + std::shared_ptr<Tensor_attributes> rng_output; + std::shared_ptr<Tensor_attributes> alibi_slopes; + int64_t alibi_slopes_size = 0; + + // Promote any 1-D seq_len / cu_seq_len / ragged-offset index tensors to the 4-D + // [n, 1, 1, 1] form the cuDNN backend requires (see promote_1d_index_tensor_to_4d). + void + promote_index_tensors_to_4d() { + for (auto& key : + {input_names::SEQ_LEN_Q, input_names::SEQ_LEN_KV, input_names::CU_SEQ_LEN_Q, input_names::CU_SEQ_LEN_KV}) { + auto const it = attributes.inputs.find(key); + if (it != attributes.inputs.end()) { + promote_1d_index_tensor_to_4d(it->second); + } + } + for (auto& [key, value] : attributes.inputs) { + CUDNN_FRONTEND_UNUSED(key); + if (value != nullptr) { + promote_1d_index_tensor_to_4d(value->get_ragged_offset()); + } + } + for (auto& [key, value] : attributes.outputs) { + CUDNN_FRONTEND_UNUSED(key); + if (value != nullptr) { + promote_1d_index_tensor_to_4d(value->get_ragged_offset()); + } + } + } + + public: + SDPA_attributes attributes; + + SDPANodeBase(SDPA_attributes&& attributes_, detail::Context const& context) + : NodeCRTP<DerivedT>(context), attributes(std::move(attributes_)) {} + + bool + is_paged_v() const { + auto page_table_v_it = attributes.inputs.find(input_names::Page_table_V); + return ((page_table_v_it) != attributes.inputs.end() && page_table_v_it->second != nullptr); + } + + bool + is_paged_k() const { + auto page_table_k_it = attributes.inputs.find(input_names::Page_table_K); + return ((page_table_k_it) != attributes.inputs.end() && page_table_k_it->second != nullptr); + } + + bool + has_seq_len_q() const { + auto seq_len_Q_it = attributes.inputs.find(SDPA_attributes::input_names::SEQ_LEN_Q); + return ((seq_len_Q_it) != attributes.inputs.end() && seq_len_Q_it->second != nullptr); + } + + bool + has_seq_len_kv() const { + auto seq_len_KV_it = attributes.inputs.find(SDPA_attributes::input_names::SEQ_LEN_KV); + return ((seq_len_KV_it) != attributes.inputs.end() && seq_len_KV_it->second != nullptr); + } + + bool + has_cu_seq_len_q() const { + auto cu_seq_len_Q_it = attributes.inputs.find(SDPA_attributes::input_names::CU_SEQ_LEN_Q); + return ((cu_seq_len_Q_it) != attributes.inputs.end() && cu_seq_len_Q_it->second != nullptr); + } + + bool + has_cu_seq_len_kv() const { + auto cu_seq_len_KV_it = attributes.inputs.find(SDPA_attributes::input_names::CU_SEQ_LEN_KV); + return ((cu_seq_len_KV_it) != attributes.inputs.end() && cu_seq_len_KV_it->second != nullptr); + } + + // Helper function to detect MXFP8 (microscaling FP8) mode + // MXFP8 uses block-wise scale factors with E8M0 data type and F8_128x4 reordering + // When detected, we use block_scale_dequantize before matmuls instead of pointwise descale after + bool + is_mxfp8_scaling() const { + auto descale_q_it = attributes.inputs.find(input_names::Descale_Q); + if (descale_q_it == attributes.inputs.end() || descale_q_it->second == nullptr) { + return false; + } + auto const& descale_q = descale_q_it->second; + return (descale_q->get_data_type() == DataType_t::FP8_E8M0 && + descale_q->get_reordering_type() == TensorReordering_t::F8_128x4); + } + + // Helper function to infer KV sequence length + // Note that it cannot be run as part of infer_properties_node as + // this is being used in pre_validate_node + int64_t + infer_s_kv() const { + int64_t s_kv = -1; + + auto get_input_dim = [this](const SDPA_attributes::input_names& input_name) { + auto const input_it = attributes.inputs.find(input_name); + if (input_it != attributes.inputs.end()) { + return input_it->second->get_dim(); + } else { + return std::vector<int64_t>({-1, -1, -1, -1}); + } + }; + + auto const& k_dim = get_input_dim(input_names::K); + auto const& v_dim = get_input_dim(input_names::V); + + // If s_kv was set explicitly, use that + if (attributes.max_seq_len_kv.has_value()) { + s_kv = attributes.max_seq_len_kv.value(); + } + // When one of K or V cache are paged, s_kv can be extracted directly + else if (!is_paged_k()) { + s_kv = k_dim[2]; + + } else if (!is_paged_v()) { + s_kv = v_dim[2]; + } else { + CUDNN_FE_LOG_LABEL_ENDL( + "WARNING: maximum kv sequence length is being inferred. To set it explicitly, please use " + "\"set_paged_attention_max_seq_len_kv\""); + + auto bias_it = attributes.inputs.find(input_names::Bias); + auto rng_it = attributes.outputs.find(output_names::RNG_DUMP); + + // If there is a bias, extract it from there + if (bias_it != attributes.inputs.end() && bias_it->second != nullptr) { + s_kv = get_input_dim(input_names::Bias)[3]; + // If there is an rng_dump output, extract it from there + } else if (rng_it != attributes.outputs.end() && rng_it->second != nullptr) { + s_kv = rng_it->second->get_dim()[3]; + // When both caches are paged, and the above failed, we need to infer s_kv from the page table and + // container + } else { + // [b, 1, ceil(s_kv/block_size), 1] + auto page_table_dim_k = get_input_dim(input_names::Page_table_K); + // [b, h_k, block_size, d_k] + auto const container_dim_k = get_input_dim(input_names::K); + int64_t s_k = page_table_dim_k[2] * container_dim_k[2]; + + // [b, 1, ceil(s_kv/block_size), 1] + auto page_table_dim_v = get_input_dim(input_names::Page_table_V); + // [b, h_v, block_size, d_v] + auto const container_dim_v = get_input_dim(input_names::V); + int64_t s_v = page_table_dim_v[2] * container_dim_v[2]; + + s_kv = std::min(s_k, s_v); + } + } + + return s_kv; + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating SDPANode " << attributes.name); + + // check that Q, K, V, O tensors has been assigned + // check that dim and strides has been assigned and last stride is 1 +#define CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(port, port_map) \ + { \ + std::shared_ptr<Tensor_attributes> tensor_ptr = port_map.at(port); \ + RETURN_CUDNN_FRONTEND_ERROR_IF(tensor_ptr->get_dim().size() != 4, \ + error_code_t::ATTRIBUTE_NOT_SET, \ + "The dim for " + std::string(#port) + " is invalid"); \ + RETURN_CUDNN_FRONTEND_ERROR_IF(tensor_ptr->get_stride().size() != 4, \ + error_code_t::ATTRIBUTE_NOT_SET, \ + "The stride for " + std::string(#port) + " is invalid"); \ + RETURN_CUDNN_FRONTEND_ERROR_IF( \ + tensor_ptr->get_stride()[3] != 1, \ + error_code_t::GRAPH_NOT_SUPPORTED, \ + "The stride for the last dimension corresponding to the embedding size per head should be 1 for " + \ + std::string(#port)); \ + } + + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::Q, attributes.inputs); + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::K, attributes.inputs); + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::V, attributes.inputs); + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(output_names::O, attributes.outputs); + + if (attributes.generate_stats.value_or(false) == true) { + CUDNN_FE_VALIDATE_OUTPUT_TENSOR(output_names::Stats); + } + + // If max is requested, validate that the output tensor is present + if (attributes.outputs.find(output_names::Max) != attributes.outputs.end() && + attributes.outputs.at(output_names::Max) != nullptr) { + CUDNN_FE_VALIDATE_OUTPUT_TENSOR(output_names::Max); + } + + // If sum_exp is requested, validate that the output tensor is present + if (attributes.outputs.find(output_names::Sum_exp) != attributes.outputs.end() && + attributes.outputs.at(output_names::Sum_exp) != nullptr) { + CUDNN_FE_VALIDATE_OUTPUT_TENSOR(output_names::Sum_exp); + } + +#undef CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE + + // validate backend limitations for the operation + auto validation_result = + attributes.validate_sdpa_support_surface(this->context, infer_s_kv(), is_paged_k(), is_paged_v()); + if (validation_result.is_good() == false) { + return validation_result; + } + + // return NOT_SET if sink_token present with 9.12 and below + RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 91300 && + attributes.inputs.find(input_names::SINK_TOKEN) != attributes.inputs.end(), + error_code_t::ATTRIBUTE_NOT_SET, + "SDPA with sink_token is not supported before 9.13."); + + // Validate MXFP8 scale factors if present + if (is_mxfp8_scaling()) { + // MXFP8 requires cuDNN 9.21.0 or later + RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 92100, + error_code_t::GRAPH_NOT_SUPPORTED, + "MXFP8 SDPA requires cuDNN 9.21.0 or later"); + + // Get device SM version from context + CHECK_CUDNN_FRONTEND_ERROR(this->context.populate_sm_version_from_device()); + int32_t const sm_version = this->context.get_sm_version(); + int32_t const prop_major = sm_version / 10; + + RETURN_CUDNN_FRONTEND_ERROR_IF(10 != prop_major, + error_code_t::GRAPH_NOT_SUPPORTED, + "MXFP8 SDPA is only supported on Blackwell Data Center architectures."); + + auto const& q_dim = attributes.inputs.at(input_names::Q)->get_dim(); + auto const& k_dim = attributes.inputs.at(input_names::K)->get_dim(); + auto const& v_dim = attributes.inputs.at(input_names::V)->get_dim(); + int64_t const b = q_dim[0]; + int64_t const h_q = q_dim[1]; + int64_t const s_q = q_dim[2]; + int64_t const d = q_dim[3]; + int64_t const h_k = k_dim[1]; + int64_t const h_v = v_dim[1]; + int64_t const s_kv = infer_s_kv(); + + // MXFP8 block size is fixed at 32 + constexpr int64_t block_size = 32; + int64_t const d_scale = (d + block_size - 1) / block_size; + int64_t const s_scale = (s_kv + block_size - 1) / block_size; + + // Validate Descale_Q + auto const& descale_q = attributes.inputs.at(input_names::Descale_Q); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_q->get_data_type() != DataType_t::FP8_E8M0, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_Q to have FP8_E8M0 data type"); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_q->get_reordering_type() != TensorReordering_t::F8_128x4, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_Q to have F8_128x4 reordering"); + RETURN_CUDNN_FRONTEND_ERROR_IF( + descale_q->get_stride()[3] != 1, + error_code_t::GRAPH_NOT_SUPPORTED, + "MXFP8 SDPA requires Descale_Q to have contiguous d_scale dimension (stride[3] == 1)"); + + // Validate Descale_K + auto const& descale_k = attributes.inputs.at(input_names::Descale_K); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_k->get_data_type() != DataType_t::FP8_E8M0, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_K to have FP8_E8M0 data type"); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_k->get_reordering_type() != TensorReordering_t::F8_128x4, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_K to have F8_128x4 reordering"); + RETURN_CUDNN_FRONTEND_ERROR_IF( + descale_k->get_stride()[3] != 1, + error_code_t::GRAPH_NOT_SUPPORTED, + "MXFP8 SDPA requires Descale_K to have contiguous d_scale dimension (stride[3] == 1)"); + + // Validate Descale_V + auto const& descale_v = attributes.inputs.at(input_names::Descale_V); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_v->get_data_type() != DataType_t::FP8_E8M0, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_V to have FP8_E8M0 data type"); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_v->get_reordering_type() != TensorReordering_t::F8_128x4, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_V to have F8_128x4 reordering"); + // // SF_V scales along s dimension (not d), so s_scale must be contiguous + // RETURN_CUDNN_FRONTEND_ERROR_IF( + // descale_v->get_stride()[2] != 1, + // error_code_t::GRAPH_NOT_SUPPORTED, + // "MXFP8 SDPA requires Descale_V to have contiguous s_scale dimension (stride[2] == 1)"); + + // Validate dimension consistency for SF_Q: [b, h_q, s_q_padded, d_scale_padded] + auto const& sf_q_dim = descale_q->get_dim(); + RETURN_CUDNN_FRONTEND_ERROR_IF(sf_q_dim[0] != b || sf_q_dim[1] != h_q, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_Q batch/head dimensions must match Q"); + RETURN_CUDNN_FRONTEND_ERROR_IF( + sf_q_dim[3] < d_scale, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_Q d_scale dimension too small (expected >= " + std::to_string(d_scale) + ")"); + + // Validate dimension consistency for SF_K: [b, h_k, s_kv_padded, d_scale_padded] + auto const& sf_k_dim = descale_k->get_dim(); + RETURN_CUDNN_FRONTEND_ERROR_IF(sf_k_dim[0] != b || sf_k_dim[1] != h_k, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_K batch/head dimensions must match K"); + RETURN_CUDNN_FRONTEND_ERROR_IF( + sf_k_dim[3] < d_scale, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_K d_scale dimension too small (expected >= " + std::to_string(d_scale) + ")"); + + // Validate dimension consistency for SF_V: [b, h_v, s_scale_padded, d_padded] + auto const& sf_v_dim = descale_v->get_dim(); + RETURN_CUDNN_FRONTEND_ERROR_IF(sf_v_dim[0] != b || sf_v_dim[1] != h_v, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_V batch/head dimensions must match V"); + RETURN_CUDNN_FRONTEND_ERROR_IF( + sf_v_dim[2] < s_scale, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_V s_scale dimension too small (expected >= " + std::to_string(s_scale) + ")"); + + // Log MXFP8 configuration for debugging + getLogger() << "[cudnn_frontend] INFO: MXFP8 SDPA configuration validated:" << std::endl + << " Q dims: [" << b << ", " << h_q << ", " << s_q << ", " << d << "]" << std::endl + << " SF_Q dims: [" << sf_q_dim[0] << ", " << sf_q_dim[1] << ", " << sf_q_dim[2] << ", " + << sf_q_dim[3] << "]" << std::endl + << " SF_K dims: [" << sf_k_dim[0] << ", " << sf_k_dim[1] << ", " << sf_k_dim[2] << ", " + << sf_k_dim[3] << "]" << std::endl + << " SF_V dims: [" << sf_v_dim[0] << ", " << sf_v_dim[1] << ", " << sf_v_dim[2] << ", " + << sf_v_dim[3] << "]" << std::endl + << " Expected d_scale: " << d_scale << ", s_scale: " << s_scale << std::endl; + } + + return {error_code_t::OK, ""}; + } + + error_t + infer_properties_node() override final { + // If a 1-D length/offset index tensor [n] is supplied, promote it to the 4-D + // [n, 1, 1, 1] (contiguous) form the cuDNN backend requires. + promote_index_tensors_to_4d(); + + if (attributes.generate_stats.value_or(false)) { + auto stats = attributes.outputs.at(output_names::Stats); + auto stats_dim = stats->get_dim(); + + if (stats_dim.empty()) { + // Fill properties of virtual tensors + auto const& p_dim = attributes.inputs[input_names::Q]->get_dim(); + auto b = p_dim[0]; + auto h = p_dim[1]; + auto s_q = p_dim[2]; + stats->set_dim({b, h, s_q, 1}).set_stride({h * s_q, s_q, 1, 1}); + } + } + + if (attributes.outputs[output_names::Max] != nullptr) { + auto max = attributes.outputs.at(output_names::Max); + + if (max->get_dim().empty()) { + // Fill properties of virtual tensors + auto const& p_dim = attributes.inputs[input_names::Q]->get_dim(); + auto b = p_dim[0]; + auto h = p_dim[1]; + auto s_q = p_dim[2]; + max->set_dim({b, h, s_q, 1}).set_stride({h * s_q, s_q, 1, 1}); + } + } + + if (attributes.outputs[output_names::Sum_exp] != nullptr) { + auto sum_exp = attributes.outputs.at(output_names::Sum_exp); + + if (sum_exp->get_dim().empty()) { + // Fill properties of virtual tensors + auto const& p_dim = attributes.inputs[input_names::Q]->get_dim(); + auto b = p_dim[0]; + auto h = p_dim[1]; + auto s_q = p_dim[2]; + sum_exp->set_dim({b, h, s_q, 1}).set_stride({h * s_q, s_q, 1, 1}); + } + } + return {error_code_t::OK, ""}; + } + + error_t + post_validate_node() const override final { +#define CUDNN_FE_VALIDATE_STRIDE(port, port_map) \ + { \ + auto const& t = port_map.find(port); \ + RETURN_CUDNN_FRONTEND_ERROR_IF( \ + t->second->get_stride().back() != 1, \ + error_code_t::GRAPH_NOT_SUPPORTED, \ + "The stride for the last dimension corresponding to the embedding size per head should be 1 for " + \ + std::string(#port)); \ + } + + CUDNN_FE_VALIDATE_STRIDE(output_names::O, attributes.outputs); + +#undef CUDNN_FE_VALIDATE_STRIDE + + return {error_code_t::OK, ""}; + } + + virtual int64_t + get_fe_workspace_size_node() const override final { + int64_t size = 0; + + // align alibi slopes memory to 16 bytes + size += ((alibi_slopes_size + 15) / 16 * 16); + + return size; + } + + virtual error_t + collect_tensors_in_workspace_node( + std::unordered_map<Tensor_attributes::uid_t, std::tuple<int64_t, int64_t, std::vector<float>>>& + workspace_modifications, + int64_t& offset) const override final { + if (attributes.alibi_mask) { + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(Q, input_names::Q); + int64_t const h_q = Q->second->get_dim()[1]; + auto alibi_slopes_vec = detail::get_alibi_slope(h_q); + workspace_modifications.emplace(alibi_slopes->get_uid(), std::make_tuple(0, offset, alibi_slopes_vec)); + int64_t alibi_slopes_size_padded = ((alibi_slopes_size + 15) / 16 * 16); + offset = offset + alibi_slopes_size_padded; + } + return {error_code_t::OK, ""}; + } + + error_t + collect_tensors_to_dump_node( + std::vector<std::pair<std::shared_ptr<Tensor_attributes>, char>>& tensors_to_dump) const override final { + std::unordered_set<Tensor_attributes::uid_t> seen_uids; + auto add_tensor = [&tensors_to_dump, &seen_uids](std::shared_ptr<Tensor_attributes> const& tensor) { + if (tensor == nullptr) { + return; + } + if (seen_uids.insert(tensor->get_uid()).second) { + tensors_to_dump.emplace_back(tensor, 'd'); + } + }; + + auto const seq_len_q_it = attributes.inputs.find(input_names::SEQ_LEN_Q); + if (seq_len_q_it != attributes.inputs.end()) { + add_tensor(seq_len_q_it->second); + } + + auto const seq_len_kv_it = attributes.inputs.find(input_names::SEQ_LEN_KV); + if (seq_len_kv_it != attributes.inputs.end()) { + add_tensor(seq_len_kv_it->second); + } + + auto const cu_seq_len_q_it = attributes.inputs.find(input_names::CU_SEQ_LEN_Q); + if (cu_seq_len_q_it != attributes.inputs.end()) { + add_tensor(cu_seq_len_q_it->second); + } + + auto const cu_seq_len_kv_it = attributes.inputs.find(input_names::CU_SEQ_LEN_KV); + if (cu_seq_len_kv_it != attributes.inputs.end()) { + add_tensor(cu_seq_len_kv_it->second); + } + + for (auto const& tensor : {attributes.inputs.at(input_names::Q), + attributes.inputs.at(input_names::K), + attributes.inputs.at(input_names::V), + attributes.outputs.at(output_names::O)}) { + if (tensor != nullptr) { + add_tensor(tensor->get_ragged_offset()); + } + } + + auto const stats_it = attributes.outputs.find(output_names::Stats); + if (stats_it != attributes.outputs.end() && stats_it->second != nullptr) { + add_tensor(stats_it->second->get_ragged_offset()); + } + + auto const max_it = attributes.outputs.find(output_names::Max); + if (max_it != attributes.outputs.end() && max_it->second != nullptr) { + add_tensor(max_it->second->get_ragged_offset()); + } + + auto const sum_exp_it = attributes.outputs.find(output_names::Sum_exp); + if (sum_exp_it != attributes.outputs.end() && sum_exp_it->second != nullptr) { + add_tensor(sum_exp_it->second->get_ragged_offset()); + } + + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j["is_mxfp8"] = is_mxfp8_scaling(); + j["unfuse_fma"] = attributes.unfuse_fma; + if (auto const rescale_threshold = get_rescale_threshold_from_env(); rescale_threshold.has_value()) { + j["rescale_threshold"] = rescale_threshold.value(); + } + if (is_mxfp8_scaling()) { + j.update(R"({"tag": "SDPA_MXFP8_FWD"})"_json); + } else if (attributes.mma_core_mode == DataType_t::FP8_E4M3 || + attributes.mma_core_mode == DataType_t::FP8_E5M2) { + j.update(R"({"tag": "SDPA_FP8_FWD"})"_json); + } else { + j.update(R"({"tag": "SDPA"})"_json); + } + } +#endif +}; + +class CompositeSDPANode : public SDPANodeBase<CompositeSDPANode> { + public: + CompositeSDPANode(SDPA_attributes&& attributes_, detail::Context const& context) + : SDPANodeBase(std::move(attributes_), context) {} + + Type + getType() override final { + return Type::COMPOSITE; + } + + error_t + expand_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for CompositeSDPANode node " << attributes.name); + + // DO NOT REMOVE + // input data type is needed for: + // - aType of bmm2 + // - dropout scale in pre 8.9.3 + attributes.fill_from_context(this->context); + + // Gather dim to fill properties of virtual tensors + auto const& q_dim = attributes.inputs[input_names::Q]->get_dim(); + auto b = q_dim[0]; + auto h_q = q_dim[1]; + auto s_q = q_dim[2]; + auto d_qk = q_dim[3]; + auto const& k_dim = attributes.inputs[input_names::K]->get_dim(); + auto h_k = k_dim[1]; + auto const& v_dim = attributes.inputs[input_names::V]->get_dim(); + auto h_v = v_dim[1]; + auto d_v = v_dim[3]; + // Infer s_kv + int64_t s_kv = infer_s_kv(); + + // Check if using MXFP8 (microscaling FP8) with block-wise scale factors + // Need to check this early because MXFP8 dequantization must happen before K transpose + bool const use_mxfp8 = is_mxfp8_scaling(); + + std::shared_ptr<Tensor_attributes> k_cache; + if (!is_paged_k()) { + if (use_mxfp8) { + // MXFP8: Transpose K and SF_K first, then dequantize + // The backend expects both K and its scale factor in transposed layout + + // Step 1: Transpose K -> KT by swapping dims and strides + // K = {b, h_k, s_kv, d_qk} -> KT = {b, h_k, d_qk, s_kv} + std::vector<int64_t> kt_dim = attributes.inputs[input_names::K]->get_dim(); + std::vector<int64_t> kt_stride = attributes.inputs[input_names::K]->get_stride(); + std::swap(kt_dim[2], kt_dim[3]); + std::swap(kt_stride[2], kt_stride[3]); + attributes.inputs[input_names::K]->set_dim(kt_dim); + attributes.inputs[input_names::K]->set_stride(kt_stride); + + // Step 2: Transpose SF_K similarly + // SF_K = {b, h_k, s_kv_scale, d_qk_scale} -> {b, h_k, d_qk_scale, s_kv_scale} + auto& sf_k = attributes.inputs[input_names::Descale_K]; + std::vector<int64_t> sf_k_dim = sf_k->get_dim(); + std::vector<int64_t> sf_k_stride = sf_k->get_stride(); + std::swap(sf_k_dim[2], sf_k_dim[3]); + std::swap(sf_k_stride[2], sf_k_stride[3]); + sf_k->set_dim(sf_k_dim); + sf_k->set_stride(sf_k_stride); + + // Step 3: Dequantize transposed K with transposed SF_K + auto dequant_k_attrs = Block_scale_dequantize_attributes() + .set_name("dequant_k") + .set_block_size({1, 32}); // Standard MXFP8 block size + + auto k_dequant = std::make_shared<Tensor_attributes>(); + k_dequant->set_is_virtual(true); + k_dequant->set_dim(kt_dim); + k_dequant->set_stride(kt_stride); + + block_scale_dequantize(attributes.inputs[input_names::K], sf_k, dequant_k_attrs, k_dequant); + + k_cache = k_dequant; + } else { + // Non-MXFP8: map K->KT directly + // cuDNN frontend API attention requires Q, K, V where + // Q = {b, h_q, s_q, d_qk} + // K = {b, h_k, s_kv, d_qk} + // V = {b, h_v, s_kv, d_v} + // but cuDNN backend API attention requires Q, KT, V + // Q = {b, h_q, s_q, d_qk} + // KT = {b, h_k, d_qk, s_kv} + // V = {b, h_v, s_kv, d_v} + // So the code below maps the K->KT + std::vector<int64_t> temp_vec; + + temp_vec = attributes.inputs[input_names::K]->get_dim(); + std::swap(temp_vec[2], temp_vec[3]); + attributes.inputs[input_names::K]->set_dim(temp_vec); + + temp_vec = attributes.inputs[input_names::K]->get_stride(); + std::swap(temp_vec[2], temp_vec[3]); + attributes.inputs[input_names::K]->set_stride(temp_vec); + + k_cache = attributes.inputs[input_names::K]; + } + } else { + // Create a paged cache load operation + auto paged_cache_load_attributes_k = PagedCacheLoad_attributes().set_name("paged_k_cache_operation"); + // Need to create virtual tensor descriptor for yOut here as it cannot be inferred + // K-cache has BHDS layout + k_cache = std::make_shared<Tensor_attributes>(); + k_cache->set_is_virtual(true); + k_cache->set_dim({b, h_k, d_qk, s_kv}); + k_cache->set_stride({d_qk * s_kv * h_k, d_qk * s_kv, 1, d_qk}); + k_cache->set_data_type(attributes.inputs[input_names::K]->get_data_type()); + paged_cache_load(attributes.inputs[input_names::K], + attributes.inputs[input_names::SEQ_LEN_KV], + attributes.inputs[input_names::Page_table_K], + paged_cache_load_attributes_k, + k_cache); + } + + // This tensor tracks the main chain of data flow + std::shared_ptr<Tensor_attributes> last_output; + + // Prepare Q and K for first matmul + // For MXFP8: Q is dequantized here, K was dequantized above before transpose + // For regular FP8: use raw tensors, descale applied after matmul + std::shared_ptr<Tensor_attributes> q_for_bmm1; + std::shared_ptr<Tensor_attributes> k_for_bmm1; + + if (use_mxfp8) { + // MXFP8: Dequantize Q using block scale factors + auto dequant_q_attrs = Block_scale_dequantize_attributes() + .set_name("dequant_q") + .set_block_size({1, 32}); // Standard MXFP8 block size + auto q_dequant = std::make_shared<Tensor_attributes>(); + q_dequant->set_is_virtual(true); + q_dequant->set_dim(attributes.inputs[input_names::Q]->get_dim()); + q_dequant->set_stride(attributes.inputs[input_names::Q]->get_stride()); + block_scale_dequantize(attributes.inputs[input_names::Q], + attributes.inputs[input_names::Descale_Q], + dequant_q_attrs, + q_dequant); + q_for_bmm1 = q_dequant; + + // K was already dequantized and transposed above + k_for_bmm1 = k_cache; + } else { + q_for_bmm1 = attributes.inputs[input_names::Q]; + k_for_bmm1 = k_cache; + } + + //// Q * K + auto bmm1_attributes = Matmul_attributes() + .set_name("bmm1") + .set_m_override(attributes.inputs[input_names::SEQ_LEN_Q]) + .set_n_override(attributes.inputs[input_names::SEQ_LEN_KV]); + + if (attributes.padding_mask) { + bmm1_attributes.set_padding(0.0); + } + + auto const& bmm1_output = matmul(q_for_bmm1, k_for_bmm1, bmm1_attributes); + // Setting dim and strides as pointwise op wont have knowledge of how to do it for mha. + bmm1_output->set_dim({b, h_q, s_q, s_kv}).set_stride({h_q * s_q * s_kv, s_q * s_kv, s_kv, 1}); + last_output = bmm1_output; + + //// Optional Attn scale + // In case user provided a scalar value, do a fused scalar. + if (attributes.attn_scale_value.has_value()) { + attributes.inputs[input_names::Attn_scale] = + std::make_shared<Tensor_attributes>(attributes.attn_scale_value.value()); + } + + // If attn scale present, add a pointwise mul node + if (attributes.inputs[input_names::Attn_scale]) { + Pointwise_attributes scale_attributes; + scale_attributes.set_name("attn_scale").set_mode(PointwiseMode_t::MUL); + auto const& attn_scale_output = + pointwise(last_output, attributes.inputs[input_names::Attn_scale], scale_attributes); + last_output = attn_scale_output; + } + + // Descale Q (only for non-MXFP8 per-tensor scaling) + // For MXFP8, descaling was done via block_scale_dequantize before matmul + if (!use_mxfp8 && attributes.inputs.find(input_names::Descale_Q) != attributes.inputs.end() && + attributes.inputs.at(input_names::Descale_Q) != nullptr) { + auto descale_q_attributes = Pointwise_attributes().set_mode(PointwiseMode_t::MUL).set_name("descale_q"); + last_output = pointwise(last_output, attributes.inputs.at(input_names::Descale_Q), descale_q_attributes); + } + + // Descale K (only for non-MXFP8 per-tensor scaling) + // For MXFP8, descaling was done via block_scale_dequantize before matmul + if (!use_mxfp8 && attributes.inputs.find(input_names::Descale_K) != attributes.inputs.end() && + attributes.inputs.at(input_names::Descale_K) != nullptr) { + auto descale_k_attributes = Pointwise_attributes().set_mode(PointwiseMode_t::MUL).set_name("descale_k"); + last_output = pointwise(last_output, attributes.inputs.at(input_names::Descale_K), descale_k_attributes); + } + + if (attributes.attention_score_modifier != nullptr) { + auto graph_ = std::make_shared<Graph>(); + std::shared_ptr<INode> node_ = std::static_pointer_cast<INode>(graph_); + node_->context = this->context; + last_output = attributes.attention_score_modifier(graph_, last_output); + sub_nodes.emplace_back(node_); + } + + // Optional bias + if (attributes.inputs.find(input_names::Bias) != attributes.inputs.end() && + attributes.inputs[input_names::Bias]) { + auto graph_ = std::make_shared<Graph>(); + std::shared_ptr<INode> node_ = std::static_pointer_cast<INode>(graph_); + node_->context = this->context; + last_output = attn::score_modifiers::bias(graph_, last_output, attributes.inputs[input_names::Bias]); + sub_nodes.emplace_back(node_); + } + + if (attributes.alibi_mask) { + auto graph_ = std::make_shared<Graph>(); + std::shared_ptr<INode> node_ = std::static_pointer_cast<INode>(graph_); + node_->context = this->context; + last_output = attn::score_modifiers::alibi_mask(graph_, last_output, alibi_slopes, h_q, alibi_slopes_size); + sub_nodes.emplace_back(node_); + } + + // There are two cases of applying padding mask + // 1. when actual seq_len is less than or equal to max_seq_len + if (attributes.padding_mask) { + auto graph_ = std::make_shared<Graph>(); + std::shared_ptr<INode> node_ = std::static_pointer_cast<INode>(graph_); + node_->context = this->context; + last_output = attn::score_modifiers::padding_mask(graph_, + last_output, + attributes.inputs[input_names::SEQ_LEN_KV], + attributes.inputs[input_names::SEQ_LEN_Q]); + sub_nodes.emplace_back(node_); + } + + // 2. (bug in cudnn backend) no padding with max_seq_len%64!=0 + if ((s_kv % 64 != 0) && (!(attributes.padding_mask)) && (detail::get_backend_version() < 90000)) { + auto col_index_attributes = + Pointwise_attributes().set_name("gen_col_index").set_mode(PointwiseMode_t::GEN_INDEX).set_axis(3); + auto col_index_output = pointwise(last_output, col_index_attributes); + // scalar seq_kv only needs to be passed in case there in no padding mask and seq_kv is not multiple of 64. + // Also future versions of cudnn will not need it, hence tensor is pre-fixed. + auto scalar_max_seq_kv = std::make_shared<Tensor_attributes>(static_cast<int32_t>(s_kv)); + + auto col_less_seq_kv_attributes = + Pointwise_attributes().set_name("col_less_seq_kv").set_mode(PointwiseMode_t::CMP_LT); + auto col_less_seq_kv_output = pointwise(col_index_output, scalar_max_seq_kv, col_less_seq_kv_attributes); + + // Lower attributes to binary select attributes + auto negative_inf_padding = + std::make_shared<Tensor_attributes>(attn::score_modifiers::get_negative_inf_value()); + auto binary_select_attributes = + Pointwise_attributes().set_name("binary_select").set_mode(PointwiseMode_t::BINARY_SELECT); + auto padding_mask_output = + pointwise(last_output, negative_inf_padding, col_less_seq_kv_output, binary_select_attributes); + last_output = padding_mask_output; + } + + // Apply (bottom-right) causal masking (with right bound) and/or set the left bound + if (attributes.left_bound.has_value() || attributes.right_bound.has_value()) { + auto graph_ = std::make_shared<Graph>(); + std::shared_ptr<INode> node_ = std::static_pointer_cast<INode>(graph_); + node_->context = this->context; + + auto s_kv_ptr = attributes.inputs.find(input_names::SEQ_LEN_KV) != attributes.inputs.end() + ? attributes.inputs[input_names::SEQ_LEN_KV] + : nullptr; + auto s_q_ptr = attributes.inputs.find(input_names::SEQ_LEN_Q) != attributes.inputs.end() + ? attributes.inputs[input_names::SEQ_LEN_Q] + : nullptr; + + last_output = attn::score_modifiers::sliding_window_mask(graph_, + last_output, + attributes.diagonal_alignment, + attributes.left_bound, + attributes.right_bound, + s_q, + s_kv, + s_q_ptr, + s_kv_ptr, + /*cu_s_q_ptr=*/nullptr, + /*cu_s_kv_ptr=*/nullptr); + sub_nodes.emplace_back(node_); + } + + // Lower attributes to softmax attributes + auto softmax_output = std::make_shared<Tensor_attributes>(); + softmax_output->set_is_virtual(true); + + auto softmax_attributes = Softmax_attributes().set_name("softmax"); + // Set sink for softmax if user has provided a sink tensor + if (attributes.inputs.find(input_names::SINK_TOKEN) != attributes.inputs.end()) { + softmax_attributes.set_sink(attributes.inputs[input_names::SINK_TOKEN]); + } + // Special non-functional-style call. Needed because output already created and provided to user. + softmax(last_output, + softmax_attributes, + softmax_output, + attributes.outputs[output_names::Stats], + attributes.outputs[output_names::Max], + attributes.outputs[output_names::Sum_exp]); + last_output = softmax_output; + + // Two cases for training: dropout present or not + bool dropout_present = false; + auto const& dropout_mask = attributes.inputs.find(input_names::Dropout_mask); + bool const is_dropout_custom = (dropout_mask != attributes.inputs.end()) && (dropout_mask->second != nullptr); + if (attributes.dropout_probability.has_value()) { + dropout_present = true; + // Special case: Skip dropout when 0.0 probability. Only do for 8.9.3 and up as rng isn't optional earlier. + if (detail::get_backend_version() > 8902 && attributes.dropout_probability.value() == 0.0) { + dropout_present = false; + } + } else if (is_dropout_custom) { + dropout_present = true; + } + + if (dropout_present) { + if (is_dropout_custom) { + auto dropout_scale_attributes = + Pointwise_attributes().set_name("dropout_scale_mul").set_mode(PointwiseMode_t::MUL); + auto const& dropout_scale_output = + pointwise(last_output, attributes.inputs[input_names::Dropout_scale], dropout_scale_attributes); + + auto mask_attributes = + Pointwise_attributes().set_name("dropout_mask_mul").set_mode(PointwiseMode_t::MUL); + auto const& dropout_mask_output = + pointwise(dropout_scale_output, dropout_mask->second, mask_attributes); + last_output = dropout_mask_output; + } else { + if (attributes.outputs[output_names::RNG_DUMP] != nullptr) { + rng_output = attributes.outputs[output_names::RNG_DUMP]; + rng(attributes.inputs[input_names::Seed], + attributes.inputs[input_names::Offset], + Rng_attributes() + .set_name("rng") + .set_distribution(RngDistribution_t::BERNOULLI) + .set_bernoulli_probability(1.0 - attributes.dropout_probability.value()), + rng_output); + } else { + rng_output = rng(attributes.inputs[input_names::Seed], + attributes.inputs[input_names::Offset], + Rng_attributes() + .set_name("rng") + .set_distribution(RngDistribution_t::BERNOULLI) + .set_bernoulli_probability(1.0 - attributes.dropout_probability.value())); + rng_output + // Hard coding dim and strides as rng output can no inputs to infer it from. + ->set_dim({b, h_q, s_q, s_kv}) + .set_stride({h_q * s_q * s_kv, s_q * s_kv, s_kv, 1}); + } + + auto mask_attributes = + Pointwise_attributes().set_name("dropout_mask_mul").set_mode(PointwiseMode_t::MUL); + auto const& dropout_mask_output = pointwise(last_output, rng_output, mask_attributes); + last_output = dropout_mask_output; + + std::shared_ptr<cudnn_frontend::graph::Tensor_attributes> dropout_scale = nullptr; + + if (detail::get_backend_version() < 8903) { + half dropout_scale_value = __float2half(1.0f / (1.0f - attributes.dropout_probability.value())); + dropout_scale = std::make_shared<Tensor_attributes>(dropout_scale_value); + } else { + float dropout_scale_value = (1.0f / (1.0f - attributes.dropout_probability.value())); + dropout_scale = std::make_shared<Tensor_attributes>(dropout_scale_value); + } + + auto dropout_scale_attributes = + Pointwise_attributes().set_name("dropout_scale").set_mode(PointwiseMode_t::MUL); + auto const& dropout_scale_output = pointwise(last_output, dropout_scale, dropout_scale_attributes); + last_output = dropout_scale_output; + } + } + + // Amax S + if (attributes.outputs.find(output_names::Amax_S) != attributes.outputs.end() && + attributes.outputs.at(output_names::Amax_S) != nullptr) { + auto amax_attributes = Reduction_attributes().set_name("amax_s").set_mode(ReductionMode_t::AMAX); + // Special non-functional-style call. Needed because output already created and provided to user. + reduction(last_output, amax_attributes, attributes.outputs.at(output_names::Amax_S)); + } + + // Scale S + if (attributes.inputs.find(input_names::Scale_S) != attributes.inputs.end() && + attributes.inputs.at(input_names::Scale_S) != nullptr) { + auto scale_s_attributes = Pointwise_attributes().set_name("scale_s").set_mode(PointwiseMode_t::MUL); + last_output = pointwise(last_output, attributes.inputs.at(input_names::Scale_S), scale_s_attributes); + } + + // Lower attributes to bmm2 attributes + // Requirement by cudnn backend to take in bmm2 aType as i/o type. + last_output->set_data_type(attributes.inputs[input_names::Q]->get_data_type()); + + auto const& seq_len_q = attributes.inputs[input_names::SEQ_LEN_Q]; + auto const& seq_len_kv = attributes.inputs[input_names::SEQ_LEN_KV]; + // auto const& V = attributes.inputs[input_names::V]; + auto const& O = attributes.outputs[output_names::O]; + + std::shared_ptr<Tensor_attributes> v_cache; + + if (!is_paged_v()) { + v_cache = attributes.inputs[input_names::V]; + } else { + auto paged_cache_load_attributes_v = PagedCacheLoad_attributes().set_name("paged_v_cache_operation"); + v_cache = std::make_shared<Tensor_attributes>(); + v_cache->set_dim({b, h_v, s_kv, d_v}) + .set_stride({d_v * s_kv * h_v, d_v * s_kv, d_v, 1}) + .set_data_type(attributes.inputs[input_names::V]->get_data_type()); + v_cache->set_is_virtual(true); + paged_cache_load(attributes.inputs[input_names::V], + attributes.inputs[input_names::SEQ_LEN_KV], + attributes.inputs[input_names::Page_table_V], + paged_cache_load_attributes_v, + v_cache); + } + + //// S * V + if (attributes.mma_core_mode == DataType_t::HALF) { + auto bmm2_attributes = + Matmul_attributes().set_name("bmm2").set_m_override(seq_len_q).set_k_override(seq_len_kv); + // Special non-functional-style call. Needed because output already created and provided to user. + matmul(last_output, v_cache, bmm2_attributes, O); + } else if (attributes.mma_core_mode == DataType_t::FP8_E4M3 || + attributes.mma_core_mode == DataType_t::FP8_E5M2) { + if (use_mxfp8) { + // MXFP8: Dequantize V using block scale factors + // SF_V should be provided with dims [b, h, s_scale, d] where s is scaled + // (different from SF_Q/SF_K which have d scaled) + // No transformation needed - use SF_V as provided + + // Dequantize V with SF_V + auto dequant_v_attrs = Block_scale_dequantize_attributes() + .set_name("dequant_v") + .set_block_size({1, 32}); // Standard MXFP8 block size + auto v_dequant = std::make_shared<Tensor_attributes>(); + v_dequant->set_is_virtual(true); + v_dequant->set_dim(v_cache->get_dim()); + v_dequant->set_stride(v_cache->get_stride()); + block_scale_dequantize(v_cache, attributes.inputs[input_names::Descale_V], dequant_v_attrs, v_dequant); + + // Use regular matmul with dequantized inputs + auto bmm2_attributes = + Matmul_attributes().set_name("bmm2").set_m_override(seq_len_q).set_k_override(seq_len_kv); + // Special non-functional-style call. Needed because output already created and provided to user. + matmul(last_output, v_dequant, bmm2_attributes, O); + + // Compute Amax_O for MXFP8 + auto const& amax_o = attributes.outputs.at(output_names::Amax_O); + if (amax_o != nullptr) { + auto amax_attributes = Reduction_attributes().set_name("amax_o").set_mode(ReductionMode_t::AMAX); + // Special non-functional-style call. Needed because output already created and provided to user. + reduction(O, amax_attributes, amax_o); + } + } else { + // Regular per-tensor FP8 scaling + auto const& descale_s = attributes.inputs.at(input_names::Descale_S); + auto const& descale_v = attributes.inputs.at(input_names::Descale_V); + auto const& scale_o = attributes.inputs.at(input_names::Scale_O); + auto const& amax_o = attributes.outputs.at(output_names::Amax_O); + + auto bmm2_attributes = + Matmul_fp8_attributes().set_name("bmm2").set_m_override(seq_len_q).set_k_override(seq_len_kv); + // Special non-functional-style call. Needed because output already created and provided to user. + matmul_fp8(last_output, v_cache, descale_s, descale_v, scale_o, bmm2_attributes, O, amax_o); + } + } else { + RETURN_CUDNN_FRONTEND_ERROR_IF(true, error_code_t::GRAPH_NOT_SUPPORTED, "Unsupported MMA core mode"); + } + + return {error_code_t::OK, ""}; + } +}; + +class CompositeSDPABackwardNode : public NodeCRTP<CompositeSDPABackwardNode> { + using input_names = SDPA_backward_attributes::input_names; + using output_names = SDPA_backward_attributes::output_names; + + private: + // non-virtual node gpu tensors + std::shared_ptr<Tensor_attributes> dQ_accum; + int64_t dQ_accum_size = 0; + std::shared_ptr<Tensor_attributes> dK_fullhead; + int64_t dK_fullhead_size = 0; + std::shared_ptr<Tensor_attributes> dV_fullhead; + int64_t dV_fullhead_size = 0; + std::shared_ptr<Tensor_attributes> softmax_sum; + int64_t softmax_sum_size = 0; + std::shared_ptr<Tensor_attributes> alibi_slopes; + int64_t alibi_slopes_size = 0; + + mutable bool has_workaround_padding_mask = false; // Will be edited in pre_validate_node() + mutable int32_t s_q_for_workaround_padding_mask = 0; // Will be edited in pre_validate_node() + mutable int32_t s_kv_for_workaround_padding_mask = 0; // Will be edited in pre_validate_node() + mutable std::shared_ptr<Tensor_attributes> + workaround_padding_mask_seq_len_q; // Will be edited in pre_validate_node() + mutable std::shared_ptr<Tensor_attributes> + workaround_padding_mask_seq_len_kv; // Will be edited in pre_validate_node() + mutable int64_t batch_size_for_workaround_padding_mask = 0; // Will be edited in pre_validate_node() + mutable bool is_deterministic_algorithm_supported_on_blackwell = false; // Will be edited in pre_validate_node() + mutable bool is_d256_on_blackwell = false; // Will be edited in pre_validate_node() + + // Promote any 1-D seq_len / ragged-offset index tensors to the 4-D + // [n, 1, 1, 1] form the cuDNN backend requires (see promote_1d_index_tensor_to_4d). + void + promote_index_tensors_to_4d() { + // TODO: Handle CU_SEQ_LEN_Q and CU_SEQ_LEN_KV once bprop supports these. + for (auto& key : {input_names::SEQ_LEN_Q, input_names::SEQ_LEN_KV}) { + auto const it = attributes.inputs.find(key); + if (it != attributes.inputs.end()) { + promote_1d_index_tensor_to_4d(it->second); + } + } + for (auto& [key, value] : attributes.inputs) { + CUDNN_FRONTEND_UNUSED(key); + if (value != nullptr) { + promote_1d_index_tensor_to_4d(value->get_ragged_offset()); + } + } + for (auto& [key, value] : attributes.outputs) { + CUDNN_FRONTEND_UNUSED(key); + if (value != nullptr) { + promote_1d_index_tensor_to_4d(value->get_ragged_offset()); + } + } + } + + public: + mutable SDPA_backward_attributes attributes; // Will be edited in pre_validate_node() for workaround padding mask + + CompositeSDPABackwardNode(SDPA_backward_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::COMPOSITE; + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating CompositeSDPABackwardNode" << attributes.name); + + // check that Q, K, V, O, stats, dO, dQ, dK, dV tensors has been assigned + // check that dim and strides has been assigned and last stride is 1 +#define CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(port, port_map) \ + { \ + std::shared_ptr<Tensor_attributes> tensor_ptr = port_map.at(port); \ + RETURN_CUDNN_FRONTEND_ERROR_IF(tensor_ptr->get_dim().size() != 4, \ + error_code_t::ATTRIBUTE_NOT_SET, \ + "The dim for " + std::string(#port) + " is invalid"); \ + RETURN_CUDNN_FRONTEND_ERROR_IF(tensor_ptr->get_stride().size() != 4, \ + error_code_t::ATTRIBUTE_NOT_SET, \ + "The stride for " + std::string(#port) + " is invalid"); \ + RETURN_CUDNN_FRONTEND_ERROR_IF( \ + tensor_ptr->get_stride()[3] != 1, \ + error_code_t::GRAPH_NOT_SUPPORTED, \ + "The stride for the last dimension corresponding to the embedding size per head should be 1 for " + \ + std::string(#port)); \ + } + + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::Q, attributes.inputs); + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::K, attributes.inputs); + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::V, attributes.inputs); + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::O, attributes.inputs); + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::Stats, attributes.inputs); + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::dO, attributes.inputs); + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(output_names::dQ, attributes.outputs); + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(output_names::dK, attributes.outputs); + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(output_names::dV, attributes.outputs); + +#undef CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE + + // validate backend limitations for the operation + // clang-format off + int64_t s_q = attributes.inputs.at(input_names::Q)->get_dim()[2]; + int64_t s_kv = attributes.inputs.at(input_names::V)->get_dim()[2]; + int64_t h_q = attributes.inputs.at(input_names::Q)->get_dim()[1]; + int64_t h_k = attributes.inputs.at(input_names::K)->get_dim()[1]; + int64_t h_v = attributes.inputs.at(input_names::V)->get_dim()[1]; + int64_t d_qk = attributes.inputs.at(input_names::Q)->get_dim()[3]; + int64_t d_v = attributes.inputs.at(input_names::V)->get_dim()[3]; + + bool const is_ragged = attributes.inputs.at(input_names::Q)->get_ragged_offset() || + attributes.inputs.at(input_names::K)->get_ragged_offset() || + attributes.inputs.at(input_names::V)->get_ragged_offset() || + attributes.inputs.at(input_names::O)->get_ragged_offset(); + + auto const& bias_mask = attributes.inputs.find(input_names::Bias); + bool const is_bias = (bias_mask != attributes.inputs.end() && bias_mask->second != nullptr); + auto const& dbias_mask = attributes.outputs.find(output_names::dBias); + bool const is_dbias = (dbias_mask != attributes.outputs.end() && dbias_mask->second != nullptr); + + auto const& dropout_mask = attributes.inputs.find(input_names::Dropout_mask); + bool const is_dropout_custom = (dropout_mask != attributes.inputs.end()) && (dropout_mask->second != nullptr); + bool const is_dropout = attributes.dropout_probability.has_value() || is_dropout_custom; + + auto const& rng_tensor = attributes.outputs.find(output_names::RNG_DUMP); + bool const is_rng = (rng_tensor != attributes.outputs.end() && rng_tensor->second != nullptr); + + // validation TODO: + // - validate stats has valid dims + // - validate Q and dQ have the same dims + + // Stop s_q = S_kv = 1 from running + RETURN_CUDNN_FRONTEND_ERROR_IF(s_q == 1 && s_kv == 1, + error_code_t::GRAPH_NOT_SUPPORTED, + "s_q = s_kv = 1 is not supported."); + + // Bug workarounds for known problematic versions (TE constraint) + RETURN_CUDNN_FRONTEND_ERROR_IF( + detail::get_backend_version() == 91000 || detail::get_backend_version() == 91001, + error_code_t::GRAPH_NOT_SUPPORTED, + "SDPA FP16/BF16 backward is not supported on cuDNN 9.10.0/9.10.1 due to known bugs. " + "Please consider upgrading to 9.10.2 or newer."); + + // 9.14.0 sliding window bug: non-causal + s_kv > 1024 + sliding window + RETURN_CUDNN_FRONTEND_ERROR_IF( + detail::get_backend_version() == 91400 && s_kv > 1024 && attributes.left_bound.has_value() && + !attributes.has_causal_like_masking(), + error_code_t::GRAPH_NOT_SUPPORTED, + "cuDNN 9.14.0 has a known bug with non-causal + s_kv > 1024 + sliding window attention. " + "Please consider upgrading to 9.14.1 or newer."); + + CHECK_CUDNN_FRONTEND_ERROR(context.populate_sm_version_from_device()); + int32_t const sm_version = context.get_sm_version(); + int32_t const prop_major = sm_version / 10; + + if (prop_major == 9) { + // validate basic dimension requirements + + if ((detail::get_backend_version() >= 91100) && (detail::get_backend_version() < 91300)) { + + if ((128 < d_qk) && (d_qk <= 192) && (64 < d_v) && (d_v <= 128)) { + + // DeepSeek case, 9.11 only supports 192 hidden dim + RETURN_CUDNN_FRONTEND_ERROR_IF( (d_v != 128) && (d_qk != 192), + error_code_t::GRAPH_NOT_SUPPORTED, + "Num hidden_dim d_v should be equal to 128 if d_qk is 192"); + } + } + + RETURN_CUDNN_FRONTEND_ERROR_IF((d_qk > 256) || (d_qk % 8 != 0) || (d_v > 256) || (d_v % 8 != 0), + error_code_t::GRAPH_NOT_SUPPORTED, + "Num hidden_dim should be less than or equal to 256 and hidden_dim should be multiple of 8"); + + } else if (prop_major == 10 && detail::get_backend_version() >= 91100) { + // validate basic dimension requirements + if (d_qk == 192) { // special case for 192 hidden dim + RETURN_CUDNN_FRONTEND_ERROR_IF( (d_v != 128), + error_code_t::GRAPH_NOT_SUPPORTED, + "Num hidden_dim d_v should be equal to 128 if d_qk is 192"); + } else if (detail::get_backend_version() >= 92300 && d_qk == 256 && d_v == 256) { + is_d256_on_blackwell = true; + attributes.is_deterministic_algorithm = true; + } else { + RETURN_CUDNN_FRONTEND_ERROR_IF((d_qk > 128) || (d_qk % 8 != 0) || (d_v > 128) || (d_v % 8 != 0), + error_code_t::GRAPH_NOT_SUPPORTED, + "Num hidden_dim should be less than or equal to 128 and hidden_dim should be multiple of 8 when d_qk != d_v"); + } + } else { + // validate basic dimension requirements + RETURN_CUDNN_FRONTEND_ERROR_IF((d_qk > 128) || (d_qk % 8 != 0) || (d_v > 128) || (d_v % 8 != 0), + error_code_t::GRAPH_NOT_SUPPORTED, + "Num hidden_dim should be less than or equal to 128 and hidden_dim should be multiple of 8"); + } + + RETURN_CUDNN_FRONTEND_ERROR_IF((attributes.attention_score_modifier != nullptr) && + (attributes.alibi_mask || attributes.padding_mask || attributes.has_causal_like_masking() || + attributes.left_bound.has_value()), error_code_t::GRAPH_NOT_SUPPORTED,"Attention score mod enabled and hence other subgraphs are disabled."); + + RETURN_CUDNN_FRONTEND_ERROR_IF((h_q % h_k != 0) || (h_q % h_v != 0), + error_code_t::GRAPH_NOT_SUPPORTED, + "For group-query attention, number of heads for key and query must be a factor of number of heads for query"); + + // validate options for attn_scale + auto const& attn_scale = attributes.inputs.find(input_names::Attn_scale); + bool const has_attn_scale = (attn_scale != attributes.inputs.end()) && (attn_scale->second != nullptr); + RETURN_CUDNN_FRONTEND_ERROR_IF(has_attn_scale && attributes.attn_scale_value.has_value(), + error_code_t::ATTRIBUTE_NOT_SET, + "attn_scale with tensor and value cannot be set at the same time."); + + // validate alibi requirements + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.alibi_mask && !(attributes.right_bound.has_value() && attributes.right_bound.value() == 0), + error_code_t::GRAPH_NOT_SUPPORTED, + "When alibi mask is used, diagonal_band_right_bound needs to be set to 0."); + + // validate options for bias mask + RETURN_CUDNN_FRONTEND_ERROR_IF(is_bias && (bias_mask->second->get_data_type() == DataType_t::BOOLEAN), + error_code_t::GRAPH_NOT_SUPPORTED, + "Bias mask data type cannot be boolean"); + + if (s_kv % 128 != 0 && attributes.padding_mask == false && is_ragged == false && detail::get_backend_version() <= 91500) { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Workaround padding mask is enabled for s_q % 128 != 0 and use_padding_mask == false and is_ragged == false"); + has_workaround_padding_mask = true; + batch_size_for_workaround_padding_mask = attributes.inputs.at(input_names::Q)->get_dim()[0]; + s_q_for_workaround_padding_mask = s_q; + s_kv_for_workaround_padding_mask = s_kv; + workaround_padding_mask_seq_len_q = std::make_shared<Tensor_attributes>(); + workaround_padding_mask_seq_len_q->set_name("workaround_padding_mask_seq_len_q").set_dim({batch_size_for_workaround_padding_mask,1,1,1}).set_stride({1,1,1,1}).set_data_type(DataType_t::INT32); + workaround_padding_mask_seq_len_kv = std::make_shared<Tensor_attributes>(); + workaround_padding_mask_seq_len_kv->set_name("workaround_padding_mask_seq_len_kv").set_dim({batch_size_for_workaround_padding_mask,1,1,1}).set_stride({1,1,1,1}).set_data_type(DataType_t::INT32); + attributes.set_padding_mask(true); + attributes.set_seq_len_q(workaround_padding_mask_seq_len_q).set_seq_len_kv(workaround_padding_mask_seq_len_kv); + } + + // validate options for padding mask + auto const& seq_len_q = attributes.inputs.find(input_names::SEQ_LEN_Q); + bool const has_seq_len_q = (seq_len_q != attributes.inputs.end()) && (seq_len_q->second != nullptr); + auto const& seq_len_kv = attributes.inputs.find(input_names::SEQ_LEN_KV); + bool const has_seq_len_kv = (seq_len_kv != attributes.inputs.end()) && (seq_len_kv->second != nullptr); + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.padding_mask && (!has_seq_len_q || !has_seq_len_kv), + error_code_t::ATTRIBUTE_NOT_SET, + "Padding mask requires seq_len_q and seq_len_kv to be set."); + RETURN_CUDNN_FRONTEND_ERROR_IF((!attributes.padding_mask && !attributes.attention_score_modifier) && (has_seq_len_q || has_seq_len_kv), + error_code_t::ATTRIBUTE_NOT_SET, + "seq_len_q and seq_len_kv needs to be set only if padding mask is enabled."); + + // validate options for max_total_seq_len + RETURN_CUDNN_FRONTEND_ERROR_IF((attributes.max_total_seq_len_q.has_value() || attributes.max_total_seq_len_kv.has_value()) && !is_ragged, + error_code_t::GRAPH_NOT_SUPPORTED, + "max_total_seq_len_q is only supported with packed layout"); + + // validate options for bottom right causal mask + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.has_causal_mask_bottom_right() && (!attributes.padding_mask) && s_q > s_kv, + error_code_t::GRAPH_NOT_SUPPORTED, + "Bottom right causal mask does not support max_s_q > max_s_kv. Please virtually slice the Q tensor and pass it as max_s_q == max_s_kv"); + + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.has_causal_mask_bottom_right() && (is_bias || attributes.alibi_mask || (is_ragged && !attributes.padding_mask) || is_dropout), + error_code_t::GRAPH_NOT_SUPPORTED, + "Bottom right causal mask is only supported with is_bias=False, is_alibi=False, is_dropout=False. Further is_ragged==True is only allowed when padding_mask=True."); + + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.has_causal_mask_bottom_right() && (detail::get_backend_version() < 90600) && ((s_q % 64 != 0) || (s_kv % 64 != 0)), + error_code_t::GRAPH_NOT_SUPPORTED, + "Bottom right causal mask is only supported with s_q multiple of 64, and s_kv multiple of 64, for cudnn version below 9.6.0"); + + // validate options for sliding window length + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.left_bound.has_value() && attributes.left_bound.value() <= 0, + error_code_t::INVALID_VALUE, + "Left bound (Sliding window length) should be greater than or equals to zero when set."); + + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.left_bound.has_value() && (s_q * attributes.left_bound.value() == s_kv * attributes.left_bound.value()) && (detail::get_backend_version() <= 90900) && (prop_major == 9) && attributes.has_causal_mask_bottom_right(), + error_code_t::GRAPH_NOT_SUPPORTED, + "On Hopper architecture, this specific combination of s_q, s_kv, and left_bound + right_bound + bottom right diagonal alignment is not supported for backend version 9.9 or below"); + + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.left_bound.has_value() && (!attributes.padding_mask) && s_q > s_kv, + error_code_t::GRAPH_NOT_SUPPORTED, + "Sliding window attention is only supported with max_s_q <= max_s_kv."); + + if ((detail::get_backend_version() >= 91002)) { + RETURN_CUDNN_FRONTEND_ERROR_IF((attributes.left_bound.has_value() || attributes.right_bound.has_value()) && ((is_ragged && !attributes.padding_mask)), + error_code_t::GRAPH_NOT_SUPPORTED, + "Left and right bounds with is_ragged==True is only allowed when padding_mask=True. And the diagonal alignment must be set."); + } else { + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.left_bound.has_value() && (! attributes.has_causal_like_masking() || is_dropout || is_bias || (is_ragged && !attributes.padding_mask)), + error_code_t::GRAPH_NOT_SUPPORTED, + "Left and right bounds are only supported with is_dropout=False, is_bias=False. Further is_ragged==True is only allowed when padding_mask=True. Lastly the diagonal alignment must be set."); + } + + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.right_bound.has_value() && attributes.right_bound.value() < 0, + error_code_t::INVALID_VALUE, + "Right bound needs to be larger than or equal to zero"); + + // validate options for dropout mask + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.dropout_probability.has_value() && is_dropout_custom, + error_code_t::ATTRIBUTE_NOT_SET, + "Using both, custom dropout mask and internal-mask generation using dropout probability, is ill-formed."); + + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.dropout_probability.has_value() && attributes.dropout_probability.value() == 1.0, + error_code_t::ATTRIBUTE_NOT_SET, + "Dropout probability cannot be 1 as corresponding scale wont be well formed."); + + // validate options for deterministic algorithm + if(attributes.is_deterministic_algorithm && (prop_major == 10)) { + RETURN_CUDNN_FRONTEND_ERROR_IF( (detail::get_backend_version() < 91800), + error_code_t::GRAPH_NOT_SUPPORTED, + "Deterministic algorithm is not supported on blackwell architecture with cudnn version below 9.18.0"); + + // dbias bias rng/dropout alibi + RETURN_CUDNN_FRONTEND_ERROR_IF(is_dbias || is_rng || is_dropout || attributes.alibi_mask, + error_code_t::GRAPH_NOT_SUPPORTED, + "Deterministic algorithm is not supported on blackwell architecture when dbias, rng/dropout, alibi is enabled"); + + is_deterministic_algorithm_supported_on_blackwell = true; + } + + if(detail::get_backend_version() >= 91801) { + RETURN_CUDNN_FRONTEND_ERROR_IF(is_ragged && (8 == prop_major || 12 == prop_major) && attributes.is_deterministic_algorithm, + error_code_t::GRAPH_NOT_SUPPORTED, + "Deterministic algorithm is not supported for bprop thd on SM8X and SM12X GPUs"); + + RETURN_CUDNN_FRONTEND_ERROR_IF(is_ragged && (8 == prop_major || 12 == prop_major) && attributes.inputs[input_names::Stats]->get_ragged_offset(), + error_code_t::GRAPH_NOT_SUPPORTED, + "Packed/ragged LSE is not supported for bprop thd on SM8X and SM12X GPUs"); + } + + // version specific validation + RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 90500 && is_dbias && attributes.padding_mask, + error_code_t::GRAPH_NOT_SUPPORTED, + "For cuDNN version below 9.5.0, dBias with variable sequence lengths is not supported"); + + RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 90500 && is_dbias && ((s_q % 64 != 0) || (s_kv % 64 != 0)), + error_code_t::GRAPH_NOT_SUPPORTED, + "For cuDNN version below 9.5.0, dBias not support s_q/s_kv which aren't multiple of 64"); + + RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 90600 && is_ragged && ((h_q != h_k) || (h_q != h_v)), + error_code_t::GRAPH_NOT_SUPPORTED, + "For cuDNN version below 9.6.0, group-query attention with raggged offset is not supported"); + + // TODO add version check once fixed + RETURN_CUDNN_FRONTEND_ERROR_IF(prop_major == 10 && is_ragged && is_dbias, + error_code_t::GRAPH_NOT_SUPPORTED, + "dbias with ragged is not supported for SM Major version 10"); + + // validate that datatype is set for the graph + RETURN_CUDNN_FRONTEND_ERROR_IF(this->context.get_intermediate_data_type() == DataType_t::NOT_SET, + error_code_t::ATTRIBUTE_NOT_SET, + "Intermediate tensor data type needs to be set as internal tensors require it."); + // If dsink is set, sink also needs to be set + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.outputs.find(output_names::DSINK_TOKEN) != attributes.outputs.end() && attributes.inputs.find(input_names::SINK_TOKEN) == attributes.inputs.end(), + error_code_t::ATTRIBUTE_NOT_SET, + "If dsink is set, sink also needs to be set."); + // clang-format on + + return {error_code_t::OK, ""}; + } + + error_t + infer_properties_node() override final { + // If a 1-D length/offset index tensor [n] is supplied, promote it to the 4-D + // [n, 1, 1, 1] (contiguous) form the cuDNN backend requires. + promote_index_tensors_to_4d(); + + // clang-format off + if (detail::get_backend_version() < 90600 && (attributes.max_total_seq_len_q.has_value() || attributes.max_total_seq_len_kv.has_value())) { + CUDNN_FE_LOG_LABEL_ENDL("WARNING: sdpa_backward.attributes.max_total_seq_len has been set, but cuDNN version is below 9.6.0 does not support max_total_seq_len_q. The workspace memory size required to execute this graph may be unexpectedly large"); + attributes.max_total_seq_len_q.reset(); + attributes.max_total_seq_len_kv.reset(); + } + + // TODO add version check once fixed + int64_t d_qk = attributes.inputs.at(input_names::Q)->get_dim()[3]; + int64_t d_v = attributes.inputs.at(input_names::V)->get_dim()[3]; + if ((attributes.max_total_seq_len_q.has_value() || attributes.max_total_seq_len_kv.has_value()) && (d_qk % 16 != 0 || d_v % 16 != 0)) { + CUDNN_FE_LOG_LABEL_ENDL("WARNING: sdpa_backward.attributes.max_total_seq_len has been set, but d is not a multiple of 16 has a known functional issue. The workspace memory size required to execute this graph may be unexpectedly large"); + attributes.max_total_seq_len_q.reset(); + attributes.max_total_seq_len_kv.reset(); + } + + + if(detail::get_backend_version() >= 91801) { + int32_t const prop_major = context.get_sm_version() / 10; + if(8 == prop_major || 12 == prop_major) { + if(attributes.max_total_seq_len_q.has_value() || attributes.max_total_seq_len_kv.has_value()) { + attributes.max_total_seq_len_q.reset(); + attributes.max_total_seq_len_kv.reset(); + CUDNN_FE_LOG_LABEL_ENDL("WARNING: sdpa_backward.attributes.max_total_seq_len has been set, but ampere style kernels have a known functional issue. The workspace memory size required to execute this graph may be unexpectedly large"); + + } + } + } + // clang-format on + + return {error_code_t::OK, ""}; + } + + error_t + expand_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for CompositeSDPABackwardNode " << attributes.name); + + attributes.fill_from_context(context); + + // Gather dim to fill properties of virtual tensors + auto const& q_dim = attributes.inputs[input_names::Q]->get_dim(); + auto b = q_dim[0]; + auto h_q = q_dim[1]; + auto s_q = q_dim[2]; + auto d_qk = q_dim[3]; + auto const& k_dim = attributes.inputs[input_names::K]->get_dim(); + auto h_k = k_dim[1]; + auto s_kv = k_dim[2]; + auto const& v_dim = attributes.inputs[input_names::V]->get_dim(); + auto h_v = v_dim[1]; + auto d_v = v_dim[3]; + + // cuDNN frontend API attention requires Q, K, V where + // Q = {b, h_q, s_q, d_qk} + // K = {b, h_k, s_kv, d_qk} + // V = {b, h_v, s_kv, d_v} + // but cuDNN backend API attention requires Q, KT, VT + // Q = {b, h_q, s_q, d_qk} + // KT = {b, h_k, d_qk, s_kv} + // VT = {b, h_v, d_v, s_kv} + // So the code below maps the K->KT and V->VT + std::vector<int64_t> temp_vec; + + temp_vec = attributes.inputs[input_names::K]->get_dim(); + std::swap(temp_vec[2], temp_vec[3]); + attributes.inputs[input_names::K]->set_dim(temp_vec); + + temp_vec = attributes.inputs[input_names::K]->get_stride(); + std::swap(temp_vec[2], temp_vec[3]); + attributes.inputs[input_names::K]->set_stride(temp_vec); + + temp_vec = attributes.inputs[input_names::V]->get_dim(); + std::swap(temp_vec[2], temp_vec[3]); + attributes.inputs[input_names::V]->set_dim(temp_vec); + + temp_vec = attributes.inputs[input_names::V]->get_stride(); + std::swap(temp_vec[2], temp_vec[3]); + attributes.inputs[input_names::V]->set_stride(temp_vec); + + std::shared_ptr<Tensor_attributes> last_output, exp_s_output, dS_output, rng_output; + + // --------------Initialize and create tensors before creating nodes-------------------- + // one_tensor is needed for non-dropout graphs + // one_tensor is passed by the node + auto one_tensor = std::make_shared<Tensor_attributes>(1.0f); + + if (attributes.attn_scale_value.has_value()) { + attributes.inputs[input_names::Attn_scale] = + std::make_shared<Tensor_attributes>(attributes.attn_scale_value.value()); + } + + // if dropout_mask is used, then the user passes scale and scale_inverse + bool is_dropout_prob = (attributes.dropout_probability.has_value()); + bool is_dropout_mask = (attributes.inputs[input_names::Dropout_mask] != nullptr); + if (is_dropout_prob) { + float dropout_scale_value = 1.0f / (1.0f - attributes.dropout_probability.value()); + float dropout_scale_inv_value = (1.0f - attributes.dropout_probability.value()); + + attributes.inputs[input_names::Dropout_scale] = std::make_shared<Tensor_attributes>(dropout_scale_value); + attributes.inputs[input_names::Dropout_scale_inv] = + std::make_shared<Tensor_attributes>(dropout_scale_inv_value); + } + + // ---------------------input tensor workarounds--------------------------- + + bool use_dp_workspace = false; + + int32_t const prop_major = context.get_sm_version() / 10; + + if (detail::get_backend_version() >= 8905 && detail::get_backend_version() < 90000) { + // workspace optimization is enabled by default when: + // 8.9.5 <= cudnn version < 9.0.0 + // device >= hopper + // batch * num_heads * seq_len_q * seq_len_kv * 2 <= dP workspace limit + // + // This following environment variable allows you to control the dP workspace limit. + // From cuDNN version 9.0.0, this option is obsolete will be ignored. + // CUDNN_FRONTEND_ATTN_DP_WORKSPACE_LIMIT=unset - enable workspace opt. until the default 256MB limit. + // CUDNN_FRONTEND_ATTN_DP_WORKSPACE_LIMIT=-1 - always enable workspace opt. + // CUDNN_FRONTEND_ATTN_DP_WORKSPACE_LIMIT=0 - always disable workspace opt. + // CUDNN_FRONTEND_ATTN_DP_WORKSPACE_LIMIT=n - enable workspace opt. until the n byte limit + + // hopper or above + if (prop_major >= 9) { + // default upper limit for workspace 256MB + int64_t max_dp_workspace_bytes = 256 * 1024 * 1024; + + // allow setting the upper limit with envvars + const char* env_dp_workspace_limit_char = get_environment("CUDNN_FRONTEND_ATTN_DP_WORKSPACE_LIMIT"); + if (env_dp_workspace_limit_char) { + char* end_ptr = nullptr; + max_dp_workspace_bytes = std::strtoll(env_dp_workspace_limit_char, &end_ptr, 10); + + if (*end_ptr != '\0') { + RETURN_CUDNN_FRONTEND_ERROR_IF(true, + error_code_t::ATTRIBUTE_NOT_SET, + "Invalid argument for CUDNN_FRONTEND_ATTN_DP_WORKSPACE_LIMIT " + "(int64_t; in bytes)"); + } + } + + int64_t workspace_s_q = ((s_q + 64 - 1) / 64) * 64; + int64_t workspace_s_kv = ((s_kv + 64 - 1) / 64) * 64; + int64_t required_dp_workspace_bytes = b * h_q * workspace_s_q * workspace_s_kv * 2; + + if (max_dp_workspace_bytes == -1) { + use_dp_workspace = true; + } else if (max_dp_workspace_bytes == 0) { + use_dp_workspace = false; + } else { + use_dp_workspace = (required_dp_workspace_bytes <= max_dp_workspace_bytes); + } + } + } + + // Force dP workspace implementation if: + // - dBias is enabled (dBias is only supported on workspace implementation) + // - the user force requests deterministic algorithm on hopper + if (attributes.outputs[output_names::dBias] || attributes.is_deterministic_algorithm) { + use_dp_workspace = true; + } + + // --------------RNG node-------------------- + + if (is_dropout_prob) { + if (attributes.outputs[output_names::RNG_DUMP] != nullptr) { + rng_output = attributes.outputs[output_names::RNG_DUMP]; + rng(attributes.inputs[input_names::Seed], + attributes.inputs[input_names::Offset], + Rng_attributes() + .set_name("rng") + .set_distribution(RngDistribution_t::BERNOULLI) + .set_bernoulli_probability(1.0f - attributes.dropout_probability.value()), + rng_output); + } else { + rng_output = rng(attributes.inputs[input_names::Seed], + attributes.inputs[input_names::Offset], + Rng_attributes() + .set_name("rng") + .set_distribution(RngDistribution_t::BERNOULLI) + .set_bernoulli_probability(1.0f - attributes.dropout_probability.value())); + rng_output->set_dim({b, h_q, s_q, s_kv}).set_stride({h_q * s_q * s_kv, s_q * s_kv, s_kv, 1}); + } + } else if (is_dropout_mask) { + rng_output = attributes.inputs[input_names::Dropout_mask]; + } + + // --------------"dO * o => softmax_sum" chain-------------------- + + // last_output = dO * O + last_output = pointwise(attributes.inputs[input_names::dO], + attributes.inputs[input_names::O], + Pointwise_attributes().set_name("mul_dO_O").set_mode(PointwiseMode_t::MUL)); + last_output->set_dim({b, h_q, s_q, d_v}).set_stride({h_q * s_q * d_v, s_q * d_v, h_q * d_v, 1}); + + // last_output = reduce(last_output, "b hq sq dv -> b hq sq 1") + last_output = + reduction(last_output, Reduction_attributes().set_name("reduce_dO_o").set_mode(ReductionMode_t::ADD)); + last_output->set_dim({b, h_q, s_q, 1}).set_stride({h_q * s_q, s_q, 1, 1}); + + if (attributes.outputs.find(output_names::DSINK_TOKEN) != attributes.outputs.end()) { + // sub_sink = sink - stats + auto sub_sink = pointwise(attributes.inputs[input_names::SINK_TOKEN], + attributes.inputs[input_names::Stats], + Pointwise_attributes().set_name("sub_sink").set_mode(PointwiseMode_t::SUB)); + + // exp_sink = exp(sub_sink) + auto exp_sink = + pointwise(sub_sink, Pointwise_attributes().set_name("exp_sink").set_mode(PointwiseMode_t::EXP)); + + // per_token_grad = exp_sink * last_output + auto per_token_grad = + pointwise(exp_sink, + last_output, + Pointwise_attributes().set_name("mul_exp_sink_last_output").set_mode(PointwiseMode_t::MUL)); + + // dSink = redduce(per_token_grad) + reduction(per_token_grad, + Reduction_attributes().set_name("reduce_per_token_grad").set_mode(ReductionMode_t::ADD), + attributes.outputs[output_names::DSINK_TOKEN]); + } + + // softmax_sum = last_output * dropout_scale + last_output = pointwise(last_output, + attributes.inputs[input_names::Dropout_scale_inv] + ? attributes.inputs[input_names::Dropout_scale_inv] + : one_tensor, + Pointwise_attributes().set_name("scale_dropout_inv").set_mode(PointwiseMode_t::MUL)); + last_output->set_dim({b, h_q, s_q, 1}).set_stride({h_q * s_q, s_q, 1, 1}); + + softmax_sum = last_output; + softmax_sum->set_is_virtual(false); + softmax_sum->set_dim({b, h_q, s_q, 1}); + softmax_sum->set_data_type(DataType_t::FLOAT); + + if (attributes.inputs[input_names::Stats]->get_ragged_offset() && attributes.max_total_seq_len_q.has_value()) { + // sized TH1 softmax_sum + softmax_sum->set_stride(attributes.inputs[input_names::Stats]->get_stride()); + softmax_sum->set_ragged_offset(attributes.inputs[input_names::Stats]->get_ragged_offset()); + softmax_sum_size = attributes.max_total_seq_len_q.value() * + (attributes.inputs[input_names::Stats]->get_stride())[2] * sizeof(float); + } else { + // sized BHS1 softmax_sum + softmax_sum->set_stride({h_q * s_q, s_q, 1, 1}); + softmax_sum_size = b * h_q * s_q * 1 * sizeof(float); + } + + // --------------"Q @ KT => exp_softmax => dV" chain-------------------- + + // s = einsum(q, k, "b hq sq dqk, b (hk g) skv dqk -> b hq sq skv", g=hq//hk) + last_output = matmul(attributes.inputs[input_names::Q], + attributes.inputs[input_names::K], + Matmul_attributes() + .set_name("matmul_Q_KT") + .set_m_override(attributes.inputs[input_names::SEQ_LEN_Q]) + .set_n_override(attributes.inputs[input_names::SEQ_LEN_KV])); + last_output->set_dim({b, h_q, s_q, s_kv}).set_stride({h_q * s_q * s_kv, s_q * s_kv, s_kv, 1}); + + // last_output = last_output * attention_scale + if (attributes.inputs[input_names::Attn_scale]) { + last_output = pointwise(last_output, + attributes.inputs[input_names::Attn_scale], + Pointwise_attributes().set_name("mul_s_attn_scale").set_mode(PointwiseMode_t::MUL)); + } + + if (attributes.attention_score_modifier != nullptr) { + auto graph_ = std::make_shared<Graph>(); + std::shared_ptr<INode> node_ = std::static_pointer_cast<INode>(graph_); + node_->context = context; + last_output = attributes.attention_score_modifier(graph_, last_output); + sub_nodes.emplace_back(node_); + } + + // (optional) last_output = last_output + bias + if (attributes.inputs.find(input_names::Bias) != attributes.inputs.end() && + attributes.inputs[input_names::Bias]) { + auto graph_ = std::make_shared<Graph>(); + std::shared_ptr<INode> node_ = std::static_pointer_cast<INode>(graph_); + node_->context = context; + last_output = attn::score_modifiers::bias(graph_, last_output, attributes.inputs[input_names::Bias]); + sub_nodes.emplace_back(node_); + } + + // (optional) last_output = last_output + alibi_mask + if (attributes.alibi_mask) { + auto graph_ = std::make_shared<Graph>(); + std::shared_ptr<INode> node_ = std::static_pointer_cast<INode>(graph_); + node_->context = context; + last_output = attn::score_modifiers::alibi_mask(graph_, last_output, alibi_slopes, h_q, alibi_slopes_size); + sub_nodes.emplace_back(node_); + } + + // last_output = last_output - stats + last_output = pointwise(last_output, + attributes.inputs[input_names::Stats], + Pointwise_attributes().set_name("sub_s_m").set_mode(PointwiseMode_t::SUB)); + + // (optional) Apply padding mask + if (attributes.padding_mask) { + auto graph_ = std::make_shared<Graph>(); + std::shared_ptr<INode> node_ = std::static_pointer_cast<INode>(graph_); + node_->context = context; + last_output = attn::score_modifiers::padding_mask(graph_, + last_output, + attributes.inputs[input_names::SEQ_LEN_KV], + attributes.inputs[input_names::SEQ_LEN_Q]); + sub_nodes.emplace_back(node_); + } + + // Explicitly put the padding value again after the stats have been loaded + if (attributes.padding_mask && detail::get_backend_version() >= 90000 && + detail::get_backend_version() < 91000) { + auto row_idx_output = pointwise(last_output, + Pointwise_attributes() + .set_name("gen_row_idx_2nd_padding") + .set_mode(PointwiseMode_t::GEN_INDEX) + .set_axis(2) + .set_compute_data_type(DataType_t::INT32)); + row_idx_output->set_data_type(DataType_t::INT32); + + auto col_idx_output = pointwise(last_output, + Pointwise_attributes() + .set_name("gen_col_idx_2nd_padding") + .set_mode(PointwiseMode_t::GEN_INDEX) + .set_axis(3) + .set_compute_data_type(DataType_t::INT32)); + col_idx_output->set_data_type(DataType_t::INT32); + + auto row_mask_output = pointwise(row_idx_output, + attributes.inputs[input_names::SEQ_LEN_Q], + Pointwise_attributes() + .set_name("lt_row_sq_2nd_padding") + .set_mode(PointwiseMode_t::CMP_LT) + .set_compute_data_type(DataType_t::BOOLEAN)); + row_mask_output->set_data_type(DataType_t::BOOLEAN); + + auto col_mask_output = pointwise(col_idx_output, + attributes.inputs[input_names::SEQ_LEN_KV], + Pointwise_attributes() + .set_name("lt_col_skv_2nd_padding") + .set_mode(PointwiseMode_t::CMP_LT) + .set_compute_data_type(DataType_t::BOOLEAN)); + col_mask_output->set_data_type(DataType_t::BOOLEAN); + + auto padding_mask_output = pointwise(row_mask_output, + col_mask_output, + Pointwise_attributes() + .set_name("and_row_col_2nd_padding") + .set_mode(PointwiseMode_t::LOGICAL_AND) + .set_compute_data_type(DataType_t::BOOLEAN)); + padding_mask_output->set_data_type(DataType_t::BOOLEAN); + auto negative_inf_padding = + std::make_shared<Tensor_attributes>(attn::score_modifiers::get_negative_inf_value()); + + last_output = pointwise( + last_output, + negative_inf_padding, + padding_mask_output, + Pointwise_attributes().set_name("select_2nd_padding").set_mode(PointwiseMode_t::BINARY_SELECT)); + } + + // Apply (bottom-right) causal masking (with right bound) and/or set the left bound + if (attributes.left_bound.has_value() || attributes.right_bound.has_value()) { + auto graph_ = std::make_shared<Graph>(); + std::shared_ptr<INode> node_ = std::static_pointer_cast<INode>(graph_); + node_->context = context; + + auto s_kv_ptr = attributes.inputs.find(input_names::SEQ_LEN_KV) != attributes.inputs.end() + ? attributes.inputs[input_names::SEQ_LEN_KV] + : nullptr; + auto s_q_ptr = attributes.inputs.find(input_names::SEQ_LEN_Q) != attributes.inputs.end() + ? attributes.inputs[input_names::SEQ_LEN_Q] + : nullptr; + + last_output = attn::score_modifiers::sliding_window_mask(graph_, + last_output, + attributes.diagonal_alignment, + attributes.left_bound, + attributes.right_bound, + s_q, + s_kv, + s_q_ptr, + s_kv_ptr, + /*cu_s_q_ptr=*/nullptr, + /*cu_s_kv_ptr=*/nullptr); + sub_nodes.emplace_back(std::move(node_)); + } + + // last_output = exp(last_output) + last_output = pointwise(last_output, Pointwise_attributes().set_name("exp_s").set_mode(PointwiseMode_t::EXP)); + + exp_s_output = last_output; + + // (optional) last_output = last_output * dropout rng_output + if (is_dropout_prob || is_dropout_mask) { + last_output = + pointwise(last_output, + rng_output, + Pointwise_attributes().set_name("mul_p_dropout_mask").set_mode(PointwiseMode_t::MUL)); + } + + // (optional) last_output = last_output * dropout_scale + if (attributes.inputs[input_names::Dropout_scale]) { + last_output = + pointwise(last_output, + attributes.inputs[input_names::Dropout_scale], + Pointwise_attributes().set_name("mul_p_dropout_scale").set_mode(PointwiseMode_t::MUL)); + } + + // dV = einsum(p, dO, "b hq sq skv", "b hq sq dv -> b hq skv dv") + // if GQA, then dV = reduce(dV, "b (hv g) skv dv -> b hv skv dv", g=hq//hv) + // as reshape + matmul + last_output = reshape(last_output, Reshape_attributes().set_name("reshape_p")); + last_output->set_dim({b, h_q, s_kv, s_q}).set_stride({h_q * s_q * s_kv, s_q * s_kv, 1, s_kv}); + last_output->set_data_type(attributes.inputs[input_names::Q]->get_data_type()); + + if (h_q == h_v) { + // for MHA + matmul(last_output, + attributes.inputs[input_names::dO], + Matmul_attributes() + .set_name("matmul_pT_dO") + .set_m_override(attributes.inputs[input_names::SEQ_LEN_KV]) + .set_k_override(attributes.inputs[input_names::SEQ_LEN_Q]), + attributes.outputs[output_names::dV]); + } else { + // for GQA and MQA + dV_fullhead = matmul(last_output, + attributes.inputs[input_names::dO], + Matmul_attributes() + .set_name("matmul_pT_dO") + .set_m_override(attributes.inputs[input_names::SEQ_LEN_KV]) + .set_k_override(attributes.inputs[input_names::SEQ_LEN_Q])); + + dV_fullhead->set_dim({b, h_q, s_kv, d_v}); + dV_fullhead->set_data_type(attributes.inputs[input_names::Q]->get_data_type()); + + if (attributes.outputs[output_names::dV]->get_ragged_offset() && + attributes.max_total_seq_len_kv.has_value()) { + // map dV strides to dV_fullhead strides + std::vector<int64_t> dV_fullhead_stride = attributes.outputs[output_names::dV]->get_stride(); + dV_fullhead_stride[2] = dV_fullhead_stride[2] * (h_q / h_v); // sequence stride + dV_fullhead_stride[0] = dV_fullhead_stride[0] * (h_q / h_v); // batch stride + dV_fullhead->set_stride(dV_fullhead_stride); + // map dV ragged offset to dV_fullhead ragged offset with implicit multiplier + // implicit multiplier = h_q / h_v + dV_fullhead->set_ragged_offset(attributes.outputs[output_names::dV]->get_ragged_offset()); + // non virtual dV full head + dV_fullhead->set_is_virtual(false); + dV_fullhead_size = attributes.max_total_seq_len_kv.value() * dV_fullhead_stride[2] * sizeof(float); + } else { + // sized BHSD dQ_accum + dV_fullhead->set_stride({h_q * s_kv * d_v, s_kv * d_v, d_v, 1}); + } + + reduction(dV_fullhead, + Reduction_attributes().set_name("red_dV_head").set_mode(ReductionMode_t::ADD), + attributes.outputs[output_names::dV]); + } + + // --------------"dO @ VT => dS_output => dK" chain-------------------- + + // dP = einsum(dO, v, "b hq sq dv, b (hv g) skv dv -> b hq sq skv", g=hq//hv) + last_output = matmul(attributes.inputs[input_names::dO], + attributes.inputs[input_names::V], + Matmul_attributes() + .set_name("matmul_dO_VT") + .set_m_override(attributes.inputs[input_names::SEQ_LEN_Q]) + .set_n_override(attributes.inputs[input_names::SEQ_LEN_KV])); + last_output->set_dim({b, h_q, s_q, s_kv}).set_stride({h_q * s_q * s_kv, s_q * s_kv, s_kv, 1}); + + // last_output = last_output(dP) * mask + if (is_dropout_prob || is_dropout_mask) { + last_output = pointwise(last_output, + rng_output, + Pointwise_attributes().set_name("dP_dropout_mask").set_mode(PointwiseMode_t::MUL)); + } + + // last_output = last_output - softmax_sum + last_output = pointwise(last_output, + softmax_sum, + Pointwise_attributes().set_name("sub_dP_softmax_sum").set_mode(PointwiseMode_t::SUB)); + + // last_output = last_output * exp_s_output + last_output = pointwise( + last_output, exp_s_output, Pointwise_attributes().set_name("mul_dP_exp_s").set_mode(PointwiseMode_t::MUL)); + + // (optional) last_output = last_output * dropout_scale + if (attributes.inputs[input_names::Dropout_scale]) { + last_output = + pointwise(last_output, + attributes.inputs[input_names::Dropout_scale], + Pointwise_attributes().set_name("mul_dS_dropout_scale").set_mode(PointwiseMode_t::MUL)); + } + + if (attributes.outputs[output_names::dBias]) { + reduction(last_output, + Reduction_attributes().set_name("red_dP_dBias").set_mode(ReductionMode_t::ADD), + attributes.outputs[output_names::dBias]); + } + + // apply the bprop of attention score modifier + if (attributes.attention_score_modifier_bprop != nullptr) { + auto graph_ = std::make_shared<Graph>(); + std::shared_ptr<INode> node_ = std::static_pointer_cast<INode>(graph_); + node_->context = context; + last_output = attributes.attention_score_modifier_bprop(graph_, last_output); + sub_nodes.emplace_back(node_); + } + + // (optional) last_output = last_output * bmm_scale + if (attributes.inputs[input_names::Attn_scale]) { + last_output = + pointwise(last_output, + attributes.inputs[input_names::Attn_scale], + Pointwise_attributes().set_name("mul_dS_attn_scale").set_mode(PointwiseMode_t::MUL)); + } + + dS_output = last_output; + + // dK = einsum(dS, Q, "b hq sq skv", "b hq sq dqk -> b hq skv dqk") + // if GQA, then dK = reduce(dK, "b (hk g) skv dqk -> b hk skv dqk", hq//hk) + // as reshape + matmul + last_output = reshape(last_output, Reshape_attributes().set_name("reshape_dS")); + last_output->set_dim({b, h_q, s_kv, s_q}).set_stride({h_q * s_q * s_kv, s_q * s_kv, 1, s_kv}); + last_output->set_data_type(attributes.inputs[input_names::Q]->get_data_type()); + + if (h_q == h_k) { + // for MHA + matmul(last_output, + attributes.inputs[input_names::Q], + Matmul_attributes() + .set_name("matmul_dST_Q") + .set_m_override(attributes.inputs[input_names::SEQ_LEN_KV]) + .set_k_override(attributes.inputs[input_names::SEQ_LEN_Q]), + attributes.outputs[output_names::dK]); + } else { + // for GQA and MQA + dK_fullhead = matmul(last_output, + attributes.inputs[input_names::Q], + Matmul_attributes() + .set_name("matmul_dST_Q") + .set_m_override(attributes.inputs[input_names::SEQ_LEN_KV]) + .set_k_override(attributes.inputs[input_names::SEQ_LEN_Q])); + + dK_fullhead->set_dim({b, h_q, s_kv, d_qk}); + dK_fullhead->set_data_type(attributes.inputs[input_names::Q]->get_data_type()); + + if (attributes.outputs[output_names::dK]->get_ragged_offset() && + attributes.max_total_seq_len_kv.has_value()) { + // sized THD dK_full_heads + // map dK strides to dK_fullhead strides + std::vector<int64_t> dK_fullhead_stride = attributes.outputs[output_names::dK]->get_stride(); + dK_fullhead_stride[0] = dK_fullhead_stride[0] * (h_q / h_k); // batch stride + dK_fullhead_stride[2] = dK_fullhead_stride[2] * (h_q / h_k); // sequence stride + dK_fullhead->set_stride(dK_fullhead_stride); + // map dK ragged offset to dK_fullhead ragged offset with implicit multiplier + // implicit multiplier = h_q / h_k + dK_fullhead->set_ragged_offset(attributes.outputs[output_names::dK]->get_ragged_offset()); + // non virtual dK full head + dK_fullhead->set_is_virtual(false); + dK_fullhead_size = attributes.max_total_seq_len_kv.value() * dK_fullhead_stride[2] * sizeof(float); + } else { + // sized BHSD dQ_accum + dK_fullhead->set_stride({h_q * s_kv * d_qk, s_kv * d_qk, d_qk, 1}); + } + + reduction(dK_fullhead, + Reduction_attributes().set_name("red_dK_head").set_mode(ReductionMode_t::ADD), + attributes.outputs[output_names::dK]); + } + + // --------------"dp_scaled @ K => dQ" chain-------------------- + + auto const& kt_dim = attributes.inputs[input_names::K]->get_dim(); + auto const& kt_stride = attributes.inputs[input_names::K]->get_stride(); + + // dQ = einsum(dS, K, "b hq sq skv, b (hk g) skv dqk -> b hq sq dqk", g=hq//hk) + // as reshape + matmul + last_output = reshape(attributes.inputs[input_names::K], Reshape_attributes().set_name("reshape_k")); + last_output->set_dim({kt_dim[0], kt_dim[1], kt_dim[3], kt_dim[2]}) + .set_stride({kt_stride[0], kt_stride[1], kt_stride[3], kt_stride[2]}); + + if (attributes.inputs[input_names::K]->get_ragged_offset() != nullptr) { + last_output->set_ragged_offset(attributes.inputs[input_names::K]->get_ragged_offset()); + } + + if (!use_dp_workspace) { + dQ_accum = std::make_shared<Tensor_attributes>(); + dQ_accum->set_is_virtual(false); + dQ_accum->set_dim({b, h_q, s_q, d_qk}); + dQ_accum->set_data_type(DataType_t::FLOAT); + + if (attributes.outputs[output_names::dQ]->get_ragged_offset() && + attributes.max_total_seq_len_q.has_value()) { + // sized THD dQ_accum + dQ_accum->set_stride(attributes.outputs[output_names::dQ]->get_stride()); + dQ_accum->set_ragged_offset(attributes.outputs[output_names::dQ]->get_ragged_offset()); + dQ_accum_size = attributes.max_total_seq_len_q.value() * + (attributes.outputs[output_names::dQ]->get_stride())[2] * sizeof(float); + } else { + // sized BHSD dQ_accum + dQ_accum->set_stride({h_q * s_q * d_qk, s_q * d_qk, d_qk, 1}); + dQ_accum_size = b * h_q * s_q * d_qk * sizeof(float); + } + + matmul(dS_output, + last_output, + Matmul_attributes() + .set_name("matmul_dS_K") + .set_m_override(attributes.inputs[input_names::SEQ_LEN_Q]) + .set_k_override(attributes.inputs[input_names::SEQ_LEN_KV]), + dQ_accum); + + pointwise(dQ_accum, + Pointwise_attributes().set_name("identity_dQ").set_mode(PointwiseMode_t::IDENTITY), + attributes.outputs[output_names::dQ]); + } else { + matmul(dS_output, + last_output, + Matmul_attributes() + .set_name("matmul_dS_K") + .set_m_override(attributes.inputs[input_names::SEQ_LEN_Q]) + .set_k_override(attributes.inputs[input_names::SEQ_LEN_KV]), + attributes.outputs[output_names::dQ]); + } + + return {error_code_t::OK, ""}; + } + + std::pair<int64_t, std::unordered_map<KnobType_t, int64_t>> + override_heuristics_query() const { + int32_t const sm_version = context.get_sm_version(); + bool const use_new_knobs = detail::get_backend_version() >= 92300; + // {128,128} bprop: tileM=3, tileN=2, kernelCfg=2(bprop warp), streamK=0, cgaM=0 + if (sm_version > 103 && (is_deterministic_algorithm_supported_on_blackwell)) { + if (use_new_knobs) { + return {17, + {{KnobType_t::TILE_M, 3}, + {KnobType_t::TILE_N, 2}, + {KnobType_t::KERNEL_CFG, 2}, + {KnobType_t::STREAM_K, 0}, + {KnobType_t::TILE_CGA_M, 0}, + {KnobType_t::STAGES, is_d256_on_blackwell ? 3 : 2}}}; + } else { + return {17, {{KnobType_t::KERNEL_CFG, 31}, {KnobType_t::STAGES, is_d256_on_blackwell ? 3 : 2}}}; + } + } else if (is_deterministic_algorithm_supported_on_blackwell) { + if (use_new_knobs) { + return {5, + {{KnobType_t::TILE_M, 3}, + {KnobType_t::TILE_N, 2}, + {KnobType_t::KERNEL_CFG, 2}, + {KnobType_t::STREAM_K, 0}, + {KnobType_t::TILE_CGA_M, 0}, + {KnobType_t::STAGES, is_d256_on_blackwell ? 3 : 2}}}; + } else { + return {5, {{KnobType_t::KERNEL_CFG, 31}, {KnobType_t::STAGES, is_d256_on_blackwell ? 3 : 2}}}; + } + } else { + return {-1, {}}; + } + } + + virtual int64_t + get_fe_workspace_size_node() const override final { + int64_t size = 0; + + size += ((alibi_slopes_size + 15) / 16 * 16); // align alibi slopes memory to 16 bytes + size += dQ_accum_size; + size += dK_fullhead_size; + size += dV_fullhead_size; + size += softmax_sum_size; + + if (has_workaround_padding_mask) { + size += batch_size_for_workaround_padding_mask * sizeof(int32_t) * 2; + } + + return size; + } + + virtual error_t + collect_tensors_in_workspace_node( + std::unordered_map<Tensor_attributes::uid_t, std::tuple<int64_t, int64_t, std::vector<float>>>& + workspace_modifications, + int64_t& offset) const override final { + if (attributes.alibi_mask) { + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(Q, input_names::Q); + int64_t const h_q = Q->second->get_dim()[1]; + auto alibi_slopes_vec = detail::get_alibi_slope(h_q); + workspace_modifications.emplace(alibi_slopes->get_uid(), std::make_tuple(0, offset, alibi_slopes_vec)); + int64_t alibi_slopes_size_padded = ((alibi_slopes_size + 15) / 16 * 16); + offset = offset + alibi_slopes_size_padded; + } + + if (dQ_accum && !dQ_accum->get_is_virtual()) { + if (detail::get_backend_version() < 90600) { + // prior to cuDNN 9.6.0, dQ_accum needed to be memset by frontend + workspace_modifications.emplace(dQ_accum->get_uid(), + std::make_tuple(1, offset, std::vector<float>{(float)dQ_accum_size})); + } else { + workspace_modifications.emplace(dQ_accum->get_uid(), std::make_tuple(2, offset, std::vector<float>())); + } + offset = offset + dQ_accum_size; + } + + if (dK_fullhead && !dK_fullhead->get_is_virtual()) { + workspace_modifications.emplace(dK_fullhead->get_uid(), std::make_tuple(2, offset, std::vector<float>())); + offset = offset + dK_fullhead_size; + } + + if (dV_fullhead && !dV_fullhead->get_is_virtual()) { + workspace_modifications.emplace(dV_fullhead->get_uid(), std::make_tuple(2, offset, std::vector<float>())); + offset = offset + dV_fullhead_size; + } + + if (softmax_sum && !softmax_sum->get_is_virtual()) { + workspace_modifications.emplace(softmax_sum->get_uid(), std::make_tuple(2, offset, std::vector<float>())); + offset = offset + softmax_sum_size; + } + + if (has_workaround_padding_mask) { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Collecting workaround padding mask tensors with batch size " + << batch_size_for_workaround_padding_mask << " with UIDs " + << workaround_padding_mask_seq_len_q->get_uid() << " and " + << workaround_padding_mask_seq_len_kv->get_uid()); + std::vector<int32_t> workaround_padding_mask_seq_len_q_vec(batch_size_for_workaround_padding_mask, + s_q_for_workaround_padding_mask); + std::vector<int32_t> workaround_padding_mask_seq_len_kv_vec(batch_size_for_workaround_padding_mask, + s_kv_for_workaround_padding_mask); + + // reinterpret_cast the int32_t vector data to float vector for workspace_modifications + std::vector<float> workaround_padding_mask_seq_len_q_vec_float( + reinterpret_cast<float*>(workaround_padding_mask_seq_len_q_vec.data()), + reinterpret_cast<float*>(workaround_padding_mask_seq_len_q_vec.data()) + + batch_size_for_workaround_padding_mask); + std::vector<float> workaround_padding_mask_seq_len_kv_vec_float( + reinterpret_cast<float*>(workaround_padding_mask_seq_len_kv_vec.data()), + reinterpret_cast<float*>(workaround_padding_mask_seq_len_kv_vec.data()) + + batch_size_for_workaround_padding_mask); + + workspace_modifications.emplace(workaround_padding_mask_seq_len_q->get_uid(), + std::make_tuple(0, offset, workaround_padding_mask_seq_len_q_vec_float)); + offset = offset + batch_size_for_workaround_padding_mask * sizeof(float); + workspace_modifications.emplace(workaround_padding_mask_seq_len_kv->get_uid(), + std::make_tuple(0, offset, workaround_padding_mask_seq_len_kv_vec_float)); + offset = offset + batch_size_for_workaround_padding_mask * sizeof(float); + } + + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"({"tag": "SDPA_BWD"})"_json); + } +#endif +}; + +class UnifiedSDPANode : public SDPANodeBase<UnifiedSDPANode> { + public: + UnifiedSDPANode(SDPA_attributes&& attributes_, detail::Context const& context) + : SDPANodeBase(std::move(attributes_), context) {} + + Type + getType() override final { + return Type::SDPA; + } + + error_t + expand_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for UnifiedSDPANode node " << attributes.name); + + // DO NOT REMOVE + // input data type is needed for: + // - aType of bmm2 + // - dropout scale in pre 8.9.3 + attributes.fill_from_context(this->context); + + //// Optional Attn scale + // In case user provided a scalar value, do a fused scalar. + if (attributes.attn_scale_value.has_value()) { + attributes.inputs[input_names::Attn_scale] = + std::make_shared<Tensor_attributes>(attributes.attn_scale_value.value()); + } + + // Optional attention score modifier + if (attributes.attention_score_modifier != nullptr) { + if (!subgraph) init_subgraph(); + subgraph_output = attributes.attention_score_modifier(subgraph, subgraph_output); + } + + // Optional bias + if (attributes.inputs.find(input_names::Bias) != attributes.inputs.end() && + attributes.inputs[input_names::Bias]) { + if (!subgraph) init_subgraph(); + subgraph_output = + attn::score_modifiers::bias(subgraph, subgraph_output, attributes.inputs[input_names::Bias]); + } + + // Optional alibi mask + if (attributes.alibi_mask) { + if (!subgraph) init_subgraph(); + auto h_q = attributes.inputs[input_names::Q]->get_dim()[1]; + subgraph_output = + attn::score_modifiers::alibi_mask(subgraph, subgraph_output, alibi_slopes, h_q, alibi_slopes_size); + } + + // Optional casual masking + if (attributes.left_bound.has_value() || attributes.right_bound.has_value()) { + if (!subgraph) init_subgraph(); + + auto s_q = attributes.inputs[input_names::Q]->get_dim()[2]; + auto s_kv = infer_s_kv(); + auto s_kv_ptr = attributes.inputs.find(input_names::SEQ_LEN_KV) != attributes.inputs.end() + ? attributes.inputs[input_names::SEQ_LEN_KV] + : nullptr; + auto s_q_ptr = attributes.inputs.find(input_names::SEQ_LEN_Q) != attributes.inputs.end() + ? attributes.inputs[input_names::SEQ_LEN_Q] + : nullptr; + auto cu_s_kv_ptr = attributes.inputs.find(input_names::CU_SEQ_LEN_KV) != attributes.inputs.end() + ? attributes.inputs[input_names::CU_SEQ_LEN_KV] + : nullptr; + auto cu_s_q_ptr = attributes.inputs.find(input_names::CU_SEQ_LEN_Q) != attributes.inputs.end() + ? attributes.inputs[input_names::CU_SEQ_LEN_Q] + : nullptr; + + subgraph_output = attn::score_modifiers::sliding_window_mask(subgraph, + subgraph_output, + attributes.diagonal_alignment, + attributes.left_bound, + attributes.right_bound, + s_q, + s_kv, + s_q_ptr, + s_kv_ptr, + cu_s_q_ptr, + cu_s_kv_ptr); + } + + if (subgraph) { + subgraph_output->set_name(attributes.name + "::subgraph_output"); + auto subgraph_node = std::static_pointer_cast<INode>(subgraph); + sub_nodes.emplace_back(subgraph_node); + } + + // For BE >= 9.21, express softmax as a UnifiedSoftmaxNode whose backend descriptor + // will be set via CUDNN_ATTR_OPERATION_SDPA_FWD_SOFTMAX_DESC. + // For older versions, stats (if present) is set directly via CUDNN_ATTR_OPERATION_SDPA_FWD_STATSDESC. + auto effective_cudnn_ver = std::min(detail::get_compiled_version(), detail::get_backend_version()); + + auto has_output = [&](auto name) { + auto it = attributes.outputs.find(name); + return it != attributes.outputs.end() && it->second != nullptr; + }; + bool has_softmax_features = has_output(output_names::Stats) || has_output(output_names::Max) || + has_output(output_names::Sum_exp) || attributes.has_sink_token(); + + if (effective_cudnn_ver >= 92100 && has_softmax_features) { + auto b = attributes.inputs[input_names::Q]->get_dim()[0]; + auto h_q = attributes.inputs[input_names::Q]->get_dim()[1]; + auto s_q = attributes.inputs[input_names::Q]->get_dim()[2]; + auto s_kv = infer_s_kv(); + + auto softmax_p = std::make_shared<Tensor_attributes>(); + softmax_p->set_is_virtual(true); + softmax_p->set_name(attributes.name + "::softmax_P"); + softmax_p->set_dim({b, h_q, s_q, s_kv}); + softmax_p->set_stride({h_q * s_q * s_kv, s_q * s_kv, s_kv, 1}); + + auto softmax_s = std::make_shared<Tensor_attributes>(); + softmax_s->set_is_virtual(true); + softmax_s->set_name(attributes.name + "::softmax_S"); + softmax_s->set_dim({b, h_q, s_q, s_kv}); + softmax_s->set_stride({h_q * s_q * s_kv, s_q * s_kv, s_kv, 1}); + + auto softmax_attrs = Softmax_attributes().set_name(attributes.name + "::softmax"); + softmax_attrs.inputs[Softmax_attributes::input_names::P] = softmax_p; + softmax_attrs.outputs[Softmax_attributes::output_names::S] = softmax_s; + + if (attributes.has_sink_token()) { + softmax_attrs.set_sink(attributes.inputs[input_names::SINK_TOKEN]); + } + if (has_output(output_names::Stats)) { + softmax_attrs.outputs[Softmax_attributes::output_names::Stats] = + attributes.outputs[output_names::Stats]; + } + if (has_output(output_names::Max)) { + softmax_attrs.outputs[Softmax_attributes::output_names::Max] = attributes.outputs[output_names::Max]; + } + if (has_output(output_names::Sum_exp)) { + softmax_attrs.outputs[Softmax_attributes::output_names::Sum_exp] = + attributes.outputs[output_names::Sum_exp]; + } + + softmax_node = std::make_shared<UnifiedSoftmaxNode>(std::move(softmax_attrs), context); + sub_nodes.emplace_back(softmax_node); + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations, + std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(operations); + CUDNN_FE_LOG_LABEL("INFO: Building UnifiedSDPANode operations " << attributes.name << " "); + auto cudnn_ver_error = error_t{error_code_t::GRAPH_NOT_SUPPORTED, "Unified SDPA node requires cuDNN 9.13.1"}; + +#if (CUDNN_VERSION >= 91301) + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(91301, cudnn_ver_error); + auto unified_sdpa_operation = + make_shared_backend_pointer((cudnnBackendDescriptorType_t)CUDNN_BACKEND_OPERATION_SDPA_FWD_DESCRIPTOR); + + auto Q = attributes.inputs.find(SDPA_attributes::input_names::Q)->second; + auto backend_q = tensors[Q->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_QDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_q)); + + auto K = attributes.inputs.find(SDPA_attributes::input_names::K)->second; + auto backend_k = tensors[K->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_KDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_k)); + + auto V = attributes.inputs.find(SDPA_attributes::input_names::V)->second; + auto backend_v = tensors[V->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_VDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_v)); + + auto O = attributes.outputs.find(SDPA_attributes::output_names::O)->second; + auto backend_o = tensors[O->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_ODESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_o)); + + if (softmax_node) { + auto softmax_ver_error = + error_t{error_code_t::GRAPH_NOT_SUPPORTED, "SOFTMAX_DESC in unified SDPA node requires cuDNN 9.21.0"}; +#if (CUDNN_VERSION >= 92100) + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(92100, softmax_ver_error); + + managed_backend_descriptor_t softmax_raw_ops; + std::unordered_set<Tensor_attributes::uid_t> softmax_uids; + std::vector<std::shared_ptr<cudnn_frontend::Operation>> softmax_ops; + CHECK_CUDNN_FRONTEND_ERROR( + softmax_node->create_cudnn_operations(softmax_uids, softmax_ops, softmax_raw_ops, tensors)); + + auto backend_softmax = softmax_raw_ops[0]->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_SOFTMAX_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_softmax)); + + uids_involved_in_operations.insert(softmax_uids.begin(), softmax_uids.end()); +#else + return softmax_ver_error; +#endif + } else { + auto stats_it = attributes.outputs.find(SDPA_attributes::output_names::Stats); + if (stats_it != attributes.outputs.end() && stats_it->second) { + auto backend_stats = tensors[stats_it->second->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_STATSDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_stats)); + } + } + + auto attn_scale_it = attributes.inputs.find(SDPA_attributes::input_names::Attn_scale); + if (attn_scale_it != attributes.inputs.end()) { + auto backend_scale = tensors[attn_scale_it->second->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_SCALEDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_scale)); + } + + auto block_mask_it = attributes.inputs.find(SDPA_attributes::input_names::Block_mask); + if (block_mask_it != attributes.inputs.end() && block_mask_it->second != nullptr) { + auto block_mask_cudnn_ver_error = + error_t{error_code_t::GRAPH_NOT_SUPPORTED, "Block mask in unified SDPA node requires cuDNN 9.14.0"}; +#if CUDNN_VERSION >= 91400 + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(91400, block_mask_cudnn_ver_error); + auto backend_block_mask = tensors[block_mask_it->second->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_BLOCK_MASK_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_block_mask)); +#else + return block_mask_cudnn_ver_error; +#endif + } + + // Variable-length and paged-attention attributes: + // - paged attention K/V page tables (needs cuDNN >= 9.15.0) + // - per-batch sequence-length tensors (SEQ_LEN_Q / SEQ_LEN_KV; needs cuDNN >= 9.15.0) + // - cumulative sequence-length tensors (CU_SEQ_LEN_Q / CU_SEQ_LEN_KV; needs cuDNN >= 9.24.0). + if (is_paged_k() || is_paged_v() || has_seq_len_q() || has_seq_len_kv() || has_cu_seq_len_q() || + has_cu_seq_len_kv()) { + auto paged_cudnn_ver_error = + error_t{error_code_t::GRAPH_NOT_SUPPORTED, + "Paged attention or per-batch sequence lengths in unified SDPA node require cuDNN 9.15.0"}; +#if (CUDNN_VERSION >= 91500) + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(91500, paged_cudnn_ver_error); + + if (is_paged_k()) { + auto page_table_K = attributes.inputs.find(SDPA_attributes::input_names::Page_table_K)->second; + auto backend_page_table_K = tensors[page_table_K->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_PAGE_TABLE_KDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_page_table_K)); + } + + if (is_paged_v()) { + auto page_table_V = attributes.inputs.find(SDPA_attributes::input_names::Page_table_V)->second; + auto backend_page_table_V = tensors[page_table_V->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_PAGE_TABLE_VDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_page_table_V)); + } + + if (has_seq_len_q()) { + auto seq_len_Q = attributes.inputs.find(SDPA_attributes::input_names::SEQ_LEN_Q)->second; + auto backend_seq_len_Q = tensors[seq_len_Q->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_SEQ_LEN_QDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_seq_len_Q)); + } + + if (has_seq_len_kv()) { + auto seq_len_KV = attributes.inputs.find(SDPA_attributes::input_names::SEQ_LEN_KV)->second; + auto backend_seq_len_KV = tensors[seq_len_KV->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_SEQ_LEN_KVDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_seq_len_KV)); + } + + if (has_cu_seq_len_q() || has_cu_seq_len_kv()) { + auto cu_seq_len_cudnn_ver_error = + error_t{error_code_t::GRAPH_NOT_SUPPORTED, + "Cumulative sequence length tensors in unified SDPA node require cuDNN 9.24.0"}; +#if (CUDNN_VERSION >= 92400) + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(92400, cu_seq_len_cudnn_ver_error); + + if (has_cu_seq_len_q()) { + auto cu_seq_len_Q = attributes.inputs.find(SDPA_attributes::input_names::CU_SEQ_LEN_Q)->second; + auto backend_cu_seq_len_Q = tensors[cu_seq_len_Q->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_CU_SEQ_LEN_QDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_cu_seq_len_Q)); + } + + if (has_cu_seq_len_kv()) { + auto cu_seq_len_KV = attributes.inputs.find(SDPA_attributes::input_names::CU_SEQ_LEN_KV)->second; + auto backend_cu_seq_len_KV = + tensors[cu_seq_len_KV->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_CU_SEQ_LEN_KVDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_cu_seq_len_KV)); + } +#else + return cu_seq_len_cudnn_ver_error; +#endif + } + + // Ignore attributes.max_seq_len_kv, because unified engine doesn't need it (it's harmless if set). + + // Ignore attributes.padding_mask, because unified engine already applies an implicit padding mask + // if either (seq_len_Q and seq_len_KV) or (cu_seq_len_Q and cu_seq_len_KV) are provided. + // We already checked in `SDPA_attributes::validate_sdpa_support_surface()` that padding_mask + // must be true if and only if one of those pairs is set, so we don't need to check it here. +#else + return paged_cudnn_ver_error; +#endif + } + + // Subgraph attributes + if (subgraph) { + auto subgraph_cudnn_ver_error = error_t{error_code_t::GRAPH_NOT_SUPPORTED, + "Subgraph attributes in unified SDPA node requires cuDNN 9.21.0"}; +#if (CUDNN_VERSION >= 92100) + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(92100, subgraph_cudnn_ver_error); + + CHECK_CUDNN_FRONTEND_ERROR(attn::score_modifiers::build_operation_subgraph(subgraph)); + auto subgraph_cudnn = std::static_pointer_cast<ICudnn>(subgraph); + auto backend_subgraph = subgraph_cudnn->get_operation_graph()->get_raw_desc(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_SUBGRAPH, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_subgraph)); + + auto subgraph_input_uid = subgraph_input->get_uid(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_SUBGRAPH_INPUT_UID, + CUDNN_TYPE_INT64, + 1, + &subgraph_input_uid)); + + auto subgraph_output_uid = subgraph_output->get_uid(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_SUBGRAPH_OUTPUT_UID, + CUDNN_TYPE_INT64, + 1, + &subgraph_output_uid)); + + // Add non-virtual uids from subgraph to uids_involved_in_operations + uids_involved_in_operations.insert(subgraph_cudnn->variant_pack_uids.begin(), + subgraph_cudnn->variant_pack_uids.end()); +#else + return subgraph_cudnn_ver_error; +#endif + } + + // Set unfuse_fma attribute (for SM100: use __fmul_rn + __fadd_rn instead of ffma2 in softmax) + if (attributes.unfuse_fma) { + auto unfuse_fma_cudnn_ver_error = + error_t{error_code_t::GRAPH_NOT_SUPPORTED, "Unfuse FMA in unified SDPA node requires cuDNN 9.21.0"}; +#if CUDNN_VERSION >= 92100 + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(92100, unfuse_fma_cudnn_ver_error); + bool unfuse_fma_value = true; + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_UNFUSE_FMA, + CUDNN_TYPE_BOOLEAN, + 1, + &unfuse_fma_value)); +#else + return unfuse_fma_cudnn_ver_error; +#endif + } + + // Dropout attributes + if (attributes.dropout_probability.has_value() && attributes.dropout_probability.value() != 0.0f) { + auto dropout_cudnn_ver_error = + error_t{error_code_t::GRAPH_NOT_SUPPORTED, "Dropout in unified SDPA node requires cuDNN 9.21.0"}; +#if (CUDNN_VERSION >= 92100) + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(92100, dropout_cudnn_ver_error); + + float dropout_prob = attributes.dropout_probability.value(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_DROPOUT_PROBABILITY, + CUDNN_TYPE_FLOAT, + 1, + &dropout_prob)); + + auto seed_it = attributes.inputs.find(SDPA_attributes::input_names::Seed); + if (seed_it != attributes.inputs.end() && seed_it->second != nullptr) { + auto backend_seed = tensors[seed_it->second->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_DROPOUT_SEED_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_seed)); + } + + auto offset_it = attributes.inputs.find(SDPA_attributes::input_names::Offset); + if (offset_it != attributes.inputs.end() && offset_it->second != nullptr) { + auto backend_offset = tensors[offset_it->second->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_DROPOUT_OFFSET_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_offset)); + } + + auto rng_dump_it = attributes.outputs.find(SDPA_attributes::output_names::RNG_DUMP); + if (rng_dump_it != attributes.outputs.end() && rng_dump_it->second != nullptr) { + auto backend_rng_dump = tensors[rng_dump_it->second->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_DROPOUT_RNG_DUMP_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_rng_dump)); + } +#else + return dropout_cudnn_ver_error; +#endif + } + + // FP8 per-tensor scaling attributes (descale Q/K/V/S, scale S/O inputs; amax S/O outputs). + if (attributes.mma_core_mode == DataType_t::FP8_E4M3 || attributes.mma_core_mode == DataType_t::FP8_E5M2) { + auto fp8_cudnn_ver_error = + error_t{error_code_t::GRAPH_NOT_SUPPORTED, "FP8/MXFP8 for the unified SDPA node requires cuDNN 9.30.0"}; +#if (CUDNN_VERSION >= 93000) + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(93000, fp8_cudnn_ver_error); + + // Wire an optional FP8 scaling/descaling tensor (from either the input or output map) to + // its backend descriptor, no-op if the caller did not provide it. + auto set_fp8_desc = [&](auto const& tensor_map, auto name, cudnnBackendAttributeName_t attr) -> error_t { + auto it = tensor_map.find(name); + if (it != tensor_map.end() && it->second != nullptr) { + auto backend_desc = tensors[it->second->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + attr, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_desc)); + } + return {error_code_t::OK, ""}; + }; + + CHECK_CUDNN_FRONTEND_ERROR(set_fp8_desc(attributes.inputs, + SDPA_attributes::input_names::Descale_Q, + CUDNN_ATTR_OPERATION_SDPA_FWD_DESCALE_QDESC)); + CHECK_CUDNN_FRONTEND_ERROR(set_fp8_desc(attributes.inputs, + SDPA_attributes::input_names::Descale_K, + CUDNN_ATTR_OPERATION_SDPA_FWD_DESCALE_KDESC)); + CHECK_CUDNN_FRONTEND_ERROR(set_fp8_desc(attributes.inputs, + SDPA_attributes::input_names::Descale_V, + CUDNN_ATTR_OPERATION_SDPA_FWD_DESCALE_VDESC)); + CHECK_CUDNN_FRONTEND_ERROR(set_fp8_desc(attributes.inputs, + SDPA_attributes::input_names::Descale_S, + CUDNN_ATTR_OPERATION_SDPA_FWD_DESCALE_SDESC)); + CHECK_CUDNN_FRONTEND_ERROR(set_fp8_desc( + attributes.inputs, SDPA_attributes::input_names::Scale_S, CUDNN_ATTR_OPERATION_SDPA_FWD_SCALE_SDESC)); + CHECK_CUDNN_FRONTEND_ERROR(set_fp8_desc( + attributes.inputs, SDPA_attributes::input_names::Scale_O, CUDNN_ATTR_OPERATION_SDPA_FWD_SCALE_ODESC)); + CHECK_CUDNN_FRONTEND_ERROR(set_fp8_desc( + attributes.outputs, SDPA_attributes::output_names::Amax_S, CUDNN_ATTR_OPERATION_SDPA_FWD_AMAX_SDESC)); + CHECK_CUDNN_FRONTEND_ERROR(set_fp8_desc( + attributes.outputs, SDPA_attributes::output_names::Amax_O, CUDNN_ATTR_OPERATION_SDPA_FWD_AMAX_ODESC)); +#else + return fp8_cudnn_ver_error; +#endif // CUDNN_VERSION >= 93000 + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(unified_sdpa_operation->get_backend_descriptor())); + + raw_operations.push_back(unified_sdpa_operation); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; +#else + CUDNN_FRONTEND_UNUSED(uids_involved_in_operations); + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FRONTEND_UNUSED(tensors); + return cudnn_ver_error; +#endif // CUDNN_VERSION >= 91301 + } + + protected: + std::shared_ptr<Graph> subgraph; // Pre-softmax subgraph + std::shared_ptr<Tensor_attributes> subgraph_input; + std::shared_ptr<Tensor_attributes> subgraph_output; + std::shared_ptr<UnifiedSoftmaxNode> softmax_node; // Softmax descriptor for BE >= 9.21 + + void + init_subgraph() { + subgraph = std::make_shared<Graph>(); + auto subgraph_node = std::static_pointer_cast<INode>(subgraph); + subgraph_node->context = context; + subgraph_input = std::make_shared<Tensor_attributes>(); + subgraph_input->set_is_virtual(true); + subgraph_input->set_name(attributes.name + "::subgraph_input"); + auto b = attributes.inputs[input_names::Q]->get_dim()[0]; + auto h_q = attributes.inputs[input_names::Q]->get_dim()[1]; + auto s_q = attributes.inputs[input_names::Q]->get_dim()[2]; + auto s_kv = infer_s_kv(); + subgraph_input->set_dim({b, h_q, s_q, s_kv}); + subgraph_input->set_stride({h_q * s_q * s_kv, s_q * s_kv, s_kv, 1}); + subgraph_output = subgraph_input; + } +}; + +} // namespace cudnn_frontend::graph diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/sdpa_fp8_bwd.h b/deps/cudnn-frontend/include/cudnn_frontend/node/sdpa_fp8_bwd.h new file mode 100644 index 0000000..0cec5a8 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/sdpa_fp8_bwd.h @@ -0,0 +1,1172 @@ +#pragma once + +#include "../../cudnn_frontend_Heuristics.h" +#include "../../cudnn_frontend_Logging.h" + +#include "../graph_helpers.h" +#include "../node_interface.h" + +#include "diagonal_band_mask.h" +#include "matmul_fp8.h" +#include "pointwise.h" +#include "reduction.h" +#include "softmax.h" +#include "block_scale_dequantize.h" +#include "scaled_dot_product_flash_attention.h" // For promote_1d_index_tensor_to_4d() + +namespace cudnn_frontend::graph { + +class SDPAFP8BackwardNode : public NodeCRTP<SDPAFP8BackwardNode> { + using input_names = SDPA_fp8_backward_attributes::input_names; + using output_names = SDPA_fp8_backward_attributes::output_names; + + private: + mutable bool is_deterministic_algorithm_supported_on_blackwell = false; // Will be edited in pre_validate_node() + + // Promote any 1-D seq_len / ragged-offset index tensors to the 4-D + // [n, 1, 1, 1] form the cuDNN backend requires (see promote_1d_index_tensor_to_4d). + void + promote_index_tensors_to_4d() { + // TODO: Handle CU_SEQ_LEN_Q and CU_SEQ_LEN_KV once fp8 bprop supports these. + for (auto& key : {input_names::SEQ_LEN_Q, input_names::SEQ_LEN_KV}) { + auto const it = attributes.inputs.find(key); + if (it != attributes.inputs.end()) { + promote_1d_index_tensor_to_4d(it->second); + } + } + for (auto& [key, value] : attributes.inputs) { + CUDNN_FRONTEND_UNUSED(key); + if (value != nullptr) { + promote_1d_index_tensor_to_4d(value->get_ragged_offset()); + } + } + for (auto& [key, value] : attributes.outputs) { + CUDNN_FRONTEND_UNUSED(key); + if (value != nullptr) { + promote_1d_index_tensor_to_4d(value->get_ragged_offset()); + } + } + } + + public: + mutable SDPA_fp8_backward_attributes + attributes; // mutable to allow auto-routing to deterministic in pre_validate_node() + + SDPAFP8BackwardNode(SDPA_fp8_backward_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::COMPOSITE; + } + + // Helper function to detect MXFP8 (microscaling FP8) mode + // MXFP8 uses block-wise scale factors with E8M0 data type and F8_128x4 reordering + // When detected, we use block_scale_dequantize before matmuls instead of pointwise descale after + bool + is_mxfp8_scaling() const { + auto descale_q_it = attributes.inputs.find(input_names::Descale_Q); + if (descale_q_it == attributes.inputs.end() || descale_q_it->second == nullptr) { + return false; + } + auto const& descale_q = descale_q_it->second; + return (descale_q->get_data_type() == DataType_t::FP8_E8M0 && + descale_q->get_reordering_type() == TensorReordering_t::F8_128x4); + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating SDPAFP8BackwardNode " << attributes.name); + + RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 90100, + error_code_t::GRAPH_NOT_SUPPORTED, + "sdpa fp8 backward operation is only supported starting cudnn 9.1.0. Please " + "consider upgrading your current version."); + + // Bug workaround for 9.10.0 (TE constraint) + RETURN_CUDNN_FRONTEND_ERROR_IF( + detail::get_backend_version() == 91000, + error_code_t::GRAPH_NOT_SUPPORTED, + "sdpa fp8 backward operation is not supported on cudnn 9.10.0 due to known bugs. " + "Please consider upgrading to 9.10.2 or newer."); + + CHECK_CUDNN_FRONTEND_ERROR(context.populate_sm_version_from_device()); + int32_t const sm_version = context.get_sm_version(); + int32_t const prop_major = sm_version / 10; + RETURN_CUDNN_FRONTEND_ERROR_IF( + prop_major < 9, + error_code_t::GRAPH_NOT_SUPPORTED, + "sdpa fp8 forward operation is only supported on Hopper architecture and newer. Please " + "consider using a newer architecture."); + + // check that Q, K, V, O, stats, dO, dQ, dK, dV tensors has been assigned + // check that dim and strides has been assigned and last stride is 1 +#define CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(port, port_map) \ + { \ + std::shared_ptr<Tensor_attributes> tensor_ptr = port_map.at(port); \ + RETURN_CUDNN_FRONTEND_ERROR_IF(tensor_ptr->get_dim().size() != 4, \ + error_code_t::ATTRIBUTE_NOT_SET, \ + "The dim for " + std::string(#port) + " is invalid"); \ + RETURN_CUDNN_FRONTEND_ERROR_IF(tensor_ptr->get_stride().size() != 4, \ + error_code_t::ATTRIBUTE_NOT_SET, \ + "The stride for " + std::string(#port) + " is invalid"); \ + RETURN_CUDNN_FRONTEND_ERROR_IF( \ + tensor_ptr->get_stride()[3] != 1, \ + error_code_t::GRAPH_NOT_SUPPORTED, \ + "The stride for the last dimension corresponding to the embedding size per head should be 1 for " + \ + std::string(#port)); \ + } + + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::Q, attributes.inputs); + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::K, attributes.inputs); + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::V, attributes.inputs); + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::O, attributes.inputs); + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::Stats, attributes.inputs); + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::dO, attributes.inputs); + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(output_names::dQ, attributes.outputs); + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(output_names::dK, attributes.outputs); + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(output_names::dV, attributes.outputs); + +#undef CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE + + // validate backend limitations for the operation + // clang-format off + int64_t s_q = attributes.inputs.at(input_names::Q)->get_dim()[2]; + int64_t s_kv = attributes.inputs.at(input_names::K)->get_dim()[2]; + int64_t h_q = attributes.inputs.at(input_names::Q)->get_dim()[1]; + int64_t h_k = attributes.inputs.at(input_names::K)->get_dim()[1]; + int64_t h_v = attributes.inputs.at(input_names::V)->get_dim()[1]; + int64_t d_qk = attributes.inputs.at(input_names::Q)->get_dim()[3]; + int64_t d_v = attributes.inputs.at(input_names::V)->get_dim()[3]; + + auto const& dq_tensor = attributes.outputs.at(output_names::dQ); + auto const& dq_data_type = dq_tensor->get_data_type(); + auto const& dk_tensor = attributes.outputs.at(output_names::dK); + auto const& dk_data_type = dk_tensor->get_data_type(); + auto const& dv_tensor = attributes.outputs.at(output_names::dV); + auto const& dv_data_type = dv_tensor->get_data_type(); + + auto const& bias_mask = attributes.inputs.find(input_names::Bias); + bool const is_bias = (bias_mask != attributes.inputs.end() && bias_mask->second != nullptr); + + auto const& dropout_mask = attributes.inputs.find(input_names::Dropout_mask); + bool const is_dropout_custom = (dropout_mask != attributes.inputs.end()) && (dropout_mask->second != nullptr); + bool const is_dropout = attributes.dropout_probability.has_value(); + bool const is_ragged = + attributes.inputs.at(input_names::Q)->get_ragged_offset() || + attributes.inputs.at(input_names::K)->get_ragged_offset() || + attributes.inputs.at(input_names::V)->get_ragged_offset() || + attributes.inputs.at(input_names::O)->get_ragged_offset() || + attributes.inputs.at(input_names::Stats)->get_ragged_offset() || + attributes.inputs.at(input_names::dO)->get_ragged_offset() || + attributes.outputs.at(output_names::dQ)->get_ragged_offset() || + attributes.outputs.at(output_names::dK)->get_ragged_offset() || + attributes.outputs.at(output_names::dV)->get_ragged_offset(); + + // validation TODO: + // - validate stats has valid dims + + RETURN_CUDNN_FRONTEND_ERROR_IF((prop_major == 9) && is_ragged, + error_code_t::GRAPH_NOT_SUPPORTED, + "sdpa fp8 backward with THD is not supported on Hopper architecture."); + + // validate basic dimension requirements + if(prop_major >= 10) { + RETURN_CUDNN_FRONTEND_ERROR_IF(((d_qk > 128) || (d_qk % 16 != 0)) && !(d_qk == 192 && d_v == 128), + error_code_t::GRAPH_NOT_SUPPORTED, + "hidden_dim d_qk shoud be less than or equal to 128 and hidden_dim d_qk should be multiple of 16 unless d_qk == 192 and d_v == 128"); + + RETURN_CUDNN_FRONTEND_ERROR_IF(((d_v > 128) || (d_v % 16 != 0)), + error_code_t::GRAPH_NOT_SUPPORTED, + "hidden_dim d_v shoud be less than or equal to 128 and hidden_dim d_v should be multiple of 16"); + } + else { + RETURN_CUDNN_FRONTEND_ERROR_IF((d_qk != 128) || (d_qk % 16 != 0) || (d_v != 128) || (d_v % 16 != 0), + error_code_t::GRAPH_NOT_SUPPORTED, + "hidden_dim shoud be equal to 128 and hidden_dim should be multiple of 16"); + } + RETURN_CUDNN_FRONTEND_ERROR_IF((h_q % h_k != 0) || (h_q % h_v != 0), + error_code_t::GRAPH_NOT_SUPPORTED, + "For group-query attention, number of heads for key and query must be a factor of number of heads for query"); + + // validate options for attn_scale + auto const& attn_scale = attributes.inputs.find(input_names::Attn_scale); + bool const has_attn_scale = (attn_scale != attributes.inputs.end()) && (attn_scale->second != nullptr); + RETURN_CUDNN_FRONTEND_ERROR_IF(has_attn_scale && attributes.attn_scale_value.has_value(), + error_code_t::ATTRIBUTE_NOT_SET, + "attn_scale with tensor and value cannot be set at the same time."); + + // validate options for bias mask + RETURN_CUDNN_FRONTEND_ERROR_IF(is_bias && (bias_mask->second->get_data_type() == DataType_t::BOOLEAN), + error_code_t::GRAPH_NOT_SUPPORTED, + "Bias mask data type cannot be boolean"); + + // validate options for padding mask + auto const& seq_len_q = attributes.inputs.find(input_names::SEQ_LEN_Q); + bool const has_seq_len_q = (seq_len_q != attributes.inputs.end()) && (seq_len_q->second != nullptr); + auto const& seq_len_kv = attributes.inputs.find(input_names::SEQ_LEN_KV); + bool const has_seq_len_kv = (seq_len_kv != attributes.inputs.end()) && (seq_len_kv->second != nullptr); + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.padding_mask && (!has_seq_len_q || !has_seq_len_kv), + error_code_t::ATTRIBUTE_NOT_SET, + "Padding mask requires seq_len_q and seq_len_kv to be set."); + RETURN_CUDNN_FRONTEND_ERROR_IF((!attributes.padding_mask) && (has_seq_len_q || has_seq_len_kv), + error_code_t::ATTRIBUTE_NOT_SET, + "seq_len_q and seq_len_kv needs to be set only if padding mask is enabled."); + + // validate options for dropout mask + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.dropout_probability.has_value() && is_dropout_custom, + error_code_t::ATTRIBUTE_NOT_SET, + "Using both, custom dropout mask and internal-mask generation using dropout probability, is ill-formed."); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.dropout_probability.has_value() && attributes.dropout_probability.value() == 1.0, + error_code_t::ATTRIBUTE_NOT_SET, + "Dropout probability cannot be 1 as corresponding scale wont be well formed."); + + // Validate options for causal_mask_bottom_right + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.has_causal_mask_bottom_right() && detail::get_backend_version() < 90700, + error_code_t::GRAPH_NOT_SUPPORTED, + "For cuDNN version below 9.7.0, bottom right causal masking is not supported."); + + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.has_causal_mask_bottom_right() && prop_major < 10, + error_code_t::GRAPH_NOT_SUPPORTED, + "sdpa fp8 forward operation is only supported on Blackwell architecture and newer. Please " + "consider using a newer architecture."); + + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.has_causal_mask_bottom_right() && s_q > s_kv, + error_code_t::GRAPH_NOT_SUPPORTED, + "Bottom right causal mask does not support s_q > s_kv. Please virtually slice the Q tensor and pass it as s_q == s_kv"); + + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.has_causal_mask_bottom_right() && (is_bias || is_dropout), + error_code_t::GRAPH_NOT_SUPPORTED, + "Bottom right causal mask is only supported with is_bias=False, is_dropout=False."); + + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.has_causal_mask_bottom_right() && ((s_q % 64 != 0) || (s_kv % 64 != 0)), + error_code_t::GRAPH_NOT_SUPPORTED, + "Bottom right causal mask is only supported with s_q multiple of 64, and s_kv multiple of 64"); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.left_bound.has_value() && (!attributes.padding_mask) && s_q > s_kv, + error_code_t::GRAPH_NOT_SUPPORTED, + "Sliding window attention is only supported with max_s_q <= max_s_kv."); + + // validate that datatype is set for the graph + RETURN_CUDNN_FRONTEND_ERROR_IF(context.get_intermediate_data_type() == DataType_t::NOT_SET, + error_code_t::ATTRIBUTE_NOT_SET, + "Intermediate tensor data type needs to be set as internal tensors require it."); + + // Auto-route to deterministic algorithm for cases where non-deterministic is not supported on Blackwell: + // 1. MXFP8 backward - requires deterministic (STAGES=2) + // 2. FP8 backward with d_qk <= 192 and d_v <= 128 - requires deterministic + bool const is_mxfp8 = is_mxfp8_scaling(); + bool const is_supported_dims = (d_qk <= 192) && (d_v <= 128); + if ((prop_major == 10) && !attributes.is_deterministic_algorithm) { + if (is_mxfp8) { + getLogger() << "[cudnn_frontend] INFO: MXFP8 SDPA backward detected on Blackwell - " + << "auto-routing to deterministic algorithm (non-deterministic not supported for MXFP8)" + << std::endl; + attributes.is_deterministic_algorithm = true; + } else if (is_supported_dims) { + getLogger() << "[cudnn_frontend] INFO: FP8 SDPA backward with d_qk=" << d_qk << ", d_v=" << d_v + << " detected on Blackwell - auto-routing to deterministic algorithm" + << std::endl; + attributes.is_deterministic_algorithm = true; + } + } + + // validate options for deterministic algorithm + if (attributes.is_deterministic_algorithm && (prop_major == 10)) { + RETURN_CUDNN_FRONTEND_ERROR_IF((detail::get_backend_version() < 91900), + error_code_t::GRAPH_NOT_SUPPORTED, + "FP8 deterministic algorithm (required for MXFP8 and d_qk=192/d_v=128) is not supported on Blackwell with cuDNN version below 9.19.0"); + + RETURN_CUDNN_FRONTEND_ERROR_IF(is_dropout, + error_code_t::GRAPH_NOT_SUPPORTED, + "FP8 deterministic algorithm (required for MXFP8 and d_qk=192/d_v=128) is not supported on Blackwell when dropout is enabled"); + + is_deterministic_algorithm_supported_on_blackwell = true; + } + + // if output data type is half or bfloat16 for any of dq, dk, dv, and version is below 9.13 or is not blackwell, return NOT_SUPPORTED + RETURN_CUDNN_FRONTEND_ERROR_IF( + (dq_data_type == DataType_t::HALF || dq_data_type == DataType_t::BFLOAT16 || + dk_data_type == DataType_t::HALF || dk_data_type == DataType_t::BFLOAT16 || + dv_data_type == DataType_t::HALF || dv_data_type == DataType_t::BFLOAT16) && + (detail::get_backend_version() < 91300 || prop_major < 10), + error_code_t::GRAPH_NOT_SUPPORTED, + "sdpa fp8 backward with HALF/BFLOAT16 output is only supported on Blackwell architecture " + "with cuDNN version 9.13.0 and newer."); + + // Validate MXFP8 scale factors if present + if (is_mxfp8_scaling()) { + // MXFP8 requires cuDNN 9.21.0 or later + RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 92100, + error_code_t::GRAPH_NOT_SUPPORTED, + "MXFP8 SDPA requires cuDNN 9.21.0 or later"); + int64_t const b = attributes.inputs.at(input_names::Q)->get_dim()[0]; + + // MXFP8 block size is fixed at 32 + constexpr int64_t block_size = 32; + int64_t const d_qk_scale = (d_qk + block_size - 1) / block_size; + int64_t const d_v_scale = (d_v + block_size - 1) / block_size; + int64_t const s_q_scale = (s_q + block_size - 1) / block_size; + int64_t const s_kv_scale = (s_kv + block_size - 1) / block_size; + + // Validate Descale_Q + auto const& descale_q = attributes.inputs.at(input_names::Descale_Q); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_q->get_data_type() != DataType_t::FP8_E8M0, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_Q to have FP8_E8M0 data type"); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_q->get_reordering_type() != TensorReordering_t::F8_128x4, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_Q to have F8_128x4 reordering"); + + // Validate Descale_Q_T + auto const& descale_q_T = attributes.inputs.at(input_names::Descale_Q_T); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_q_T->get_data_type() != DataType_t::FP8_E8M0, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_Q_T to have FP8_E8M0 data type"); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_q_T->get_reordering_type() != TensorReordering_t::F8_128x4, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_Q_T to have F8_128x4 reordering"); + + // Validate Descale_K + auto const& descale_k = attributes.inputs.at(input_names::Descale_K); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_k->get_data_type() != DataType_t::FP8_E8M0, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_K to have FP8_E8M0 data type"); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_k->get_reordering_type() != TensorReordering_t::F8_128x4, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_K to have F8_128x4 reordering"); + + // Validate Descale_K_T + auto const& descale_k_T = attributes.inputs.at(input_names::Descale_K_T); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_k_T->get_data_type() != DataType_t::FP8_E8M0, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_K_T to have FP8_E8M0 data type"); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_k_T->get_reordering_type() != TensorReordering_t::F8_128x4, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_K_T to have F8_128x4 reordering"); + + // Validate Descale_V + auto const& descale_v = attributes.inputs.at(input_names::Descale_V); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_v->get_data_type() != DataType_t::FP8_E8M0, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_V to have FP8_E8M0 data type"); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_v->get_reordering_type() != TensorReordering_t::F8_128x4, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_V to have F8_128x4 reordering"); + + // Validate Descale_dO + auto const& descale_do = attributes.inputs.at(input_names::Descale_dO); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_do->get_data_type() != DataType_t::FP8_E8M0, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_dO to have FP8_E8M0 data type"); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_do->get_reordering_type() != TensorReordering_t::F8_128x4, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_dO to have F8_128x4 reordering"); + + // Validate Descale_dO_T + auto const& descale_do_T = attributes.inputs.at(input_names::Descale_dO_T); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_do_T->get_data_type() != DataType_t::FP8_E8M0, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_dO_T to have FP8_E8M0 data type"); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_do_T->get_reordering_type() != TensorReordering_t::F8_128x4, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_dO_T to have F8_128x4 reordering"); + + // Validate dimension consistency for SF_Q: [b, h_q, s_q_padded, d_scale_padded] + auto const& sf_q_dim = descale_q->get_dim(); + RETURN_CUDNN_FRONTEND_ERROR_IF(sf_q_dim[0] != b || sf_q_dim[1] != h_q, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_Q batch/head dimensions must match Q"); + RETURN_CUDNN_FRONTEND_ERROR_IF( + sf_q_dim[3] < d_qk_scale, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_Q d_scale dimension too small (expected >= " + std::to_string(d_qk_scale) + ")"); + + // Validate dimension consistency for SF_Q_T: [b, h_q, s_q_scale_padded, d_padded] + auto const& sf_q_T_dim = descale_q_T->get_dim(); + RETURN_CUDNN_FRONTEND_ERROR_IF(sf_q_T_dim[0] != b || sf_q_T_dim[1] != h_q, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_Q_T batch/head dimensions must match Q"); + RETURN_CUDNN_FRONTEND_ERROR_IF( + sf_q_T_dim[2] < s_q_scale, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_Q_T s_scale dimension too small (expected >= " + std::to_string(s_q_scale) + ")"); + + // Validate dimension consistency for SF_K: [b, h_k, s_kv_padded, d_qk_scale_padded] + auto const& sf_k_dim = descale_k->get_dim(); + RETURN_CUDNN_FRONTEND_ERROR_IF(sf_k_dim[0] != b || sf_k_dim[1] != h_k, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_K batch/head dimensions must match K"); + RETURN_CUDNN_FRONTEND_ERROR_IF( + sf_k_dim[3] < d_qk_scale, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_K d_scale dimension too small (expected >= " + std::to_string(d_qk_scale) + ")"); + + // Validate dimension consistency for SF_K_T: [b, h_k, s_kv_scale_padded, d_qk_padded] + auto const& sf_k_T_dim = descale_k_T->get_dim(); + RETURN_CUDNN_FRONTEND_ERROR_IF(sf_k_T_dim[0] != b || sf_k_T_dim[1] != h_k, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_K_T batch/head dimensions must match K"); + RETURN_CUDNN_FRONTEND_ERROR_IF( + sf_k_T_dim[2] < s_kv_scale, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_K_T s_scale dimension too small (expected >= " + std::to_string(s_kv_scale) + ")"); + + // Validate dimension consistency for SF_V: [b, h_v, s_kv_scale_padded, d_v_padded] + auto const& sf_v_dim = descale_v->get_dim(); + RETURN_CUDNN_FRONTEND_ERROR_IF(sf_v_dim[0] != b || sf_v_dim[1] != h_v, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_V batch/head dimensions must match V"); + RETURN_CUDNN_FRONTEND_ERROR_IF( + sf_v_dim[2] < s_kv_scale, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_V s_scale dimension too small (expected >= " + std::to_string(s_kv_scale) + ")"); + + // Validate dimension consistency for SF_dO: [b, h_q, s_q_padded, d_v_scale_padded] + auto const& sf_do_dim = descale_do->get_dim(); + RETURN_CUDNN_FRONTEND_ERROR_IF(sf_do_dim[0] != b || sf_do_dim[1] != h_q, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_dO batch/head dimensions must match dO"); + RETURN_CUDNN_FRONTEND_ERROR_IF( + sf_do_dim[3] < d_v_scale, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_dO d_scale dimension too small (expected >= " + std::to_string(d_v_scale) + ")"); + + // Validate dimension consistency for SF_dO_T: [b, h_q, s_q_scale_padded, d_scale_padded] + auto const& sf_do_T_dim = descale_do_T->get_dim(); + RETURN_CUDNN_FRONTEND_ERROR_IF(sf_do_T_dim[0] != b || sf_do_T_dim[1] != h_q, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_dO_T batch/head dimensions must match dO"); + RETURN_CUDNN_FRONTEND_ERROR_IF( + sf_do_T_dim[2] < s_q_scale, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_dO_T s_scale dimension too small (expected >= " + std::to_string(s_q_scale) + ")"); + + // Log MXFP8 configuration for debugging + getLogger() << "[cudnn_frontend] INFO: MXFP8 SDPA configuration validated:" << std::endl + << " SF_Q dims: [" << sf_q_dim[0] << ", " << sf_q_dim[1] << ", " << sf_q_dim[2] << ", " + << sf_q_dim[3] << "]" << std::endl + << " SF_Q_T dims: [" << sf_q_T_dim[0] << ", " << sf_q_T_dim[1] << ", " << sf_q_T_dim[2] << ", " + << sf_q_T_dim[3] << "]" << std::endl + << " SF_K dims: [" << sf_k_dim[0] << ", " << sf_k_dim[1] << ", " << sf_k_dim[2] << ", " + << sf_k_dim[3] << "]" << std::endl + << " SF_K_T dims: [" << sf_k_T_dim[0] << ", " << sf_k_T_dim[1] << ", " << sf_k_T_dim[2] << ", " + << sf_k_T_dim[3] << "]" << std::endl + << " SF_V dims: [" << sf_v_dim[0] << ", " << sf_v_dim[1] << ", " << sf_v_dim[2] << ", " + << sf_v_dim[3] << "]" << std::endl + << " SF_dO dims: [" << sf_do_dim[0] << ", " << sf_do_dim[1] << ", " << sf_do_dim[2] << ", " + << sf_do_dim[3] << "]" << std::endl + << " SF_dO_T dims: [" << sf_do_T_dim[0] << ", " << sf_do_T_dim[1] << ", " << sf_do_T_dim[2] << ", " + << sf_do_T_dim[3] << "]" << std::endl + << " Expected d_qk_scale: " << d_qk_scale << ", d_v_scale: " << d_v_scale + << ", s_q_scale: " << s_q_scale << ", s_kv_scale: " << s_kv_scale << std::endl; + } + + // validate options for sink token + auto const& sink_token = attributes.inputs.find(input_names::SINK_TOKEN); + bool const has_sink_token = (sink_token != attributes.inputs.end()) && (sink_token->second != nullptr); + auto const& dsink_token = attributes.outputs.find(output_names::DSINK_TOKEN); + bool const has_dsink_token = (dsink_token != attributes.outputs.end()) && (dsink_token->second != nullptr); + RETURN_CUDNN_FRONTEND_ERROR_IF(has_dsink_token && !has_sink_token, + error_code_t::ATTRIBUTE_NOT_SET, + "If dSink_token output is requested, sink_token input must also be set."); + + return {error_code_t::OK, ""}; + } + + error_t + infer_properties_node() override final { + // If a 1-D length/offset index tensor [n] is supplied, promote it to the 4-D + // [n, 1, 1, 1] form the cuDNN backend requires. + promote_index_tensors_to_4d(); + + return {error_code_t::OK, ""}; + } + + error_t + expand_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for Scaled_dot_product_flash_attention node " + << attributes.name); + + attributes.fill_from_context(context); + + // Gather dim to fill properties of virtual tensors + auto const& q_dim = attributes.inputs[input_names::Q]->get_dim(); + auto b = q_dim[0]; + auto h_q = q_dim[1]; + auto s_q = q_dim[2]; + auto d_qk = q_dim[3]; + auto const& k_dim = attributes.inputs[input_names::K]->get_dim(); + auto h_k = k_dim[1]; + auto s_kv = k_dim[2]; + auto const& v_dim = attributes.inputs[input_names::V]->get_dim(); + auto h_v = v_dim[1]; + auto d_v = v_dim[3]; + + bool const use_mxfp8 = is_mxfp8_scaling(); + std::shared_ptr<Tensor_attributes> eff_Q, eff_K, eff_V, eff_dO; + + // cuDNN frontend API attention requires Q, K, V where + // Q = {b, h_q, s_q, d_qk} + // K = {b, h_k, s_kv, d_qk} + // V = {b, h_v, s_kv, d_v} + // but cuDNN backend API attention requires Q, KT, VT + // Q = {b, h_q, s_q, d_qk} + // KT = {b, h_k, d_qk, s_kv} + // VT = {b, h_v, d_v, s_kv} + // So the code below maps the K->KT and V->VT + if (use_mxfp8) { + // MXFP8 2-kernel bprop: Each tensor is dequantized separately for each matmul it participates in. + // Q, K, dO each need two dequantizations with different scale factors. + // O and dO are used directly (as f16) for the dO*O computation without dequantization. + + // K for BMM1 (Q@K.T): Transpose K and SF_K, then dequantize + std::vector<int64_t> kt_dim = attributes.inputs[input_names::K]->get_dim(); + std::vector<int64_t> kt_stride = attributes.inputs[input_names::K]->get_stride(); + std::swap(kt_dim[2], kt_dim[3]); + std::swap(kt_stride[2], kt_stride[3]); + attributes.inputs[input_names::K]->set_dim(kt_dim); + attributes.inputs[input_names::K]->set_stride(kt_stride); + + auto& sf_k = attributes.inputs[input_names::Descale_K]; + std::vector<int64_t> sf_k_dim = sf_k->get_dim(); + std::vector<int64_t> sf_k_stride = sf_k->get_stride(); + std::swap(sf_k_dim[2], sf_k_dim[3]); + std::swap(sf_k_stride[2], sf_k_stride[3]); + sf_k->set_dim(sf_k_dim); + sf_k->set_stride(sf_k_stride); + + auto dequant_k_dqk_attrs = Block_scale_dequantize_attributes() + .set_name("DQ_K_dqk") + .set_block_size({1, 32}); + auto k_dqk_dequant = std::make_shared<Tensor_attributes>(); + k_dqk_dequant->set_is_virtual(true); + k_dqk_dequant->set_dim(kt_dim); + k_dqk_dequant->set_stride(kt_stride); + block_scale_dequantize(attributes.inputs[input_names::K], sf_k, dequant_k_dqk_attrs, k_dqk_dequant); + eff_K = k_dqk_dequant; + + // V for dO@V.T: Transpose V and SF_V, then dequantize + std::vector<int64_t> vt_dim = attributes.inputs[input_names::V]->get_dim(); + std::vector<int64_t> vt_stride = attributes.inputs[input_names::V]->get_stride(); + std::swap(vt_dim[2], vt_dim[3]); + std::swap(vt_stride[2], vt_stride[3]); + attributes.inputs[input_names::V]->set_dim(vt_dim); + attributes.inputs[input_names::V]->set_stride(vt_stride); + + auto& sf_v = attributes.inputs[input_names::Descale_V]; + std::vector<int64_t> sf_v_dim = sf_v->get_dim(); + std::vector<int64_t> sf_v_stride = sf_v->get_stride(); + std::swap(sf_v_dim[2], sf_v_dim[3]); + std::swap(sf_v_stride[2], sf_v_stride[3]); + sf_v->set_dim(sf_v_dim); + sf_v->set_stride(sf_v_stride); + + auto dequant_v_dv_attrs = Block_scale_dequantize_attributes() + .set_name("DQ_V_dv") + .set_block_size({1, 32}); + auto v_dv_dequant = std::make_shared<Tensor_attributes>(); + v_dv_dequant->set_is_virtual(true); + v_dv_dequant->set_dim(vt_dim); + v_dv_dequant->set_stride(vt_stride); + block_scale_dequantize(attributes.inputs[input_names::V], sf_v, dequant_v_dv_attrs, v_dv_dequant); + eff_V = v_dv_dequant; + + // Q for BMM1 (Q@K.T): Dequantize Q with Descale_Q + auto dequant_q_dqk_attrs = Block_scale_dequantize_attributes() + .set_name("DQ_Q_dqk") + .set_block_size({1, 32}); + auto q_dqk_dequant = std::make_shared<Tensor_attributes>(); + q_dqk_dequant->set_is_virtual(true); + q_dqk_dequant->set_dim(attributes.inputs[input_names::Q]->get_dim()); + q_dqk_dequant->set_stride(attributes.inputs[input_names::Q]->get_stride()); + block_scale_dequantize(attributes.inputs[input_names::Q], + attributes.inputs[input_names::Descale_Q], + dequant_q_dqk_attrs, + q_dqk_dequant); + eff_Q = q_dqk_dequant; + + // dO for dO@V.T: Dequantize dO with Descale_dO + auto dequant_dO_dv_attrs = Block_scale_dequantize_attributes() + .set_name("DQ_dO_dv") + .set_block_size({1, 32}); + auto dO_dv_dequant = std::make_shared<Tensor_attributes>(); + dO_dv_dequant->set_is_virtual(true); + dO_dv_dequant->set_dim(attributes.inputs[input_names::dO]->get_dim()); + dO_dv_dequant->set_stride(attributes.inputs[input_names::dO]->get_stride()); + block_scale_dequantize(attributes.inputs[input_names::dO], + attributes.inputs[input_names::Descale_dO], + dequant_dO_dv_attrs, + dO_dv_dequant); + eff_dO = dO_dv_dequant; + + // Note: O and dO for the dO*O computation are used as raw f16 inputs (no MXFP8 dequant). + // The separate dequantizations for dO_seq, K_seq, Q_seq are done inline at their usage sites. + } else { + // Non-MXFP8 + std::vector<int64_t> temp_vec; + + temp_vec = attributes.inputs[input_names::K]->get_dim(); + std::swap(temp_vec[2], temp_vec[3]); + attributes.inputs[input_names::K]->set_dim(temp_vec); + + temp_vec = attributes.inputs[input_names::K]->get_stride(); + std::swap(temp_vec[2], temp_vec[3]); + attributes.inputs[input_names::K]->set_stride(temp_vec); + + temp_vec = attributes.inputs[input_names::V]->get_dim(); + std::swap(temp_vec[2], temp_vec[3]); + attributes.inputs[input_names::V]->set_dim(temp_vec); + + temp_vec = attributes.inputs[input_names::V]->get_stride(); + std::swap(temp_vec[2], temp_vec[3]); + attributes.inputs[input_names::V]->set_stride(temp_vec); + + eff_Q = attributes.inputs[input_names::Q]; + eff_K = attributes.inputs[input_names::K]; + eff_V = attributes.inputs[input_names::V]; + eff_dO = attributes.inputs[input_names::dO]; + } + + std::shared_ptr<Tensor_attributes> rng_output; + + auto mul_attributes = Pointwise_attributes().set_mode(PointwiseMode_t::MUL); + + // if dropout_prob is used, then the node passes scale and scale inverse + // if dropout_mask is used, then the user passes scale and scale_inverse + bool is_dropout_prob = (attributes.dropout_probability.has_value()); + bool is_dropout_mask = (attributes.inputs[input_names::Dropout_mask] != nullptr); + if (is_dropout_prob) { + float dropout_scale_value = 1.0f / (1.0f - attributes.dropout_probability.value()); + float dropout_scale_inv_value = (1.0f - attributes.dropout_probability.value()); + + attributes.inputs[input_names::Dropout_scale] = std::make_shared<Tensor_attributes>(dropout_scale_value); + attributes.inputs[input_names::Dropout_scale_inv] = + std::make_shared<Tensor_attributes>(dropout_scale_inv_value); + } + + // --------------RNG node-------------------- + + if (is_dropout_prob) { + rng_output = rng(attributes.inputs[input_names::Seed], + attributes.inputs[input_names::Offset], + Rng_attributes() + .set_name("rng") + .set_distribution(RngDistribution_t::BERNOULLI) + .set_bernoulli_probability(1.0f - attributes.dropout_probability.value())); + rng_output->set_dim({b, h_q, s_q, s_kv}).set_stride({h_q * s_q * s_kv, s_q * s_kv, s_kv, 1}); + } else if (is_dropout_mask) { + rng_output = attributes.inputs[input_names::Dropout_mask]; + } + + //// dO * O + // For MXFP8: dO_f16 and O (f16) are used directly (no block dequantization needed) + // For non-MXFP8 (17-arg API): dO and O (FP8) are used (per-tensor descale applied after reduction) + mul_attributes.set_name("mul_dO_O"); + auto last_output = use_mxfp8 + ? pointwise(attributes.inputs[input_names::dO_f16], attributes.inputs[input_names::O], mul_attributes) + : pointwise(attributes.inputs[input_names::dO], attributes.inputs[input_names::O], mul_attributes); + + // reduce(dO) + last_output = + reduction(last_output, Reduction_attributes().set_name("reduce_dO").set_mode(ReductionMode_t::ADD)); + last_output->set_dim({b, h_q, s_q, 1}).set_stride({h_q * s_q, s_q, 1, 1}); + + // Descale dO (only for non-MXFP8 per-tensor scaling) + if (!use_mxfp8) { + mul_attributes.set_name("descale_dO"); + last_output = pointwise(last_output, attributes.inputs.at(input_names::Descale_dO), mul_attributes); + last_output->set_dim({b, h_q, s_q, 1}).set_stride({h_q * s_q, s_q, 1, 1}); + } + + // Descale O (only for non-MXFP8 per-tensor scaling) + if (!use_mxfp8) { + mul_attributes.set_name("descale_O"); + last_output = pointwise(last_output, attributes.inputs.at(input_names::Descale_O), mul_attributes); + } + + // dSink_token computation: dSink = sum(exp(sink - stats) * D) over batch and sequence + // Note: The backend kernel applies the negative sign internally. + auto const& sink_token_it = attributes.inputs.find(input_names::SINK_TOKEN); + auto const& dsink_token_it = attributes.outputs.find(output_names::DSINK_TOKEN); + if (dsink_token_it != attributes.outputs.end() && dsink_token_it->second != nullptr && + sink_token_it != attributes.inputs.end() && sink_token_it->second != nullptr) { + // sub_sink = sink - stats + auto sub_sink = pointwise(sink_token_it->second, + attributes.inputs[input_names::Stats], + Pointwise_attributes().set_name("sub_sink_stats").set_mode(PointwiseMode_t::SUB)); + sub_sink->set_dim({b, h_q, s_q, 1}).set_stride({h_q * s_q, s_q, 1, 1}); + + // exp_sink = exp(sub_sink) + auto exp_sink = + pointwise(sub_sink, Pointwise_attributes().set_name("exp_sink").set_mode(PointwiseMode_t::EXP)); + exp_sink->set_dim({b, h_q, s_q, 1}).set_stride({h_q * s_q, s_q, 1, 1}); + + // per_token_grad = exp_sink * last_output (D) + auto per_token_grad = pointwise( + exp_sink, last_output, Pointwise_attributes().set_name("mul_exp_sink_D").set_mode(PointwiseMode_t::MUL)); + per_token_grad->set_dim({b, h_q, s_q, 1}).set_stride({h_q * s_q, s_q, 1, 1}); + + // dSink = reduce(per_token_grad) over batch and sequence dimensions + reduction(per_token_grad, + Reduction_attributes().set_name("reduce_dSink").set_mode(ReductionMode_t::ADD), + dsink_token_it->second); + } + + // softmax_sum = last_output * dropout_scale + // Note: When sink_token is enabled and dropout is NOT present, we still need a MUL operation + // in the softmax path so the backend pattern matcher (fortAttentionBackwardRuntimeFusionEngine) + // can correctly distinguish between the dSink chain (MUL→REDUCE) and the softmax path (MUL→SUB). + // Without this, the fallback at lines 3786-3796 incorrectly identifies dSink MUL as the softmax path. + bool const has_sink_token = (sink_token_it != attributes.inputs.end() && sink_token_it->second != nullptr); + if (attributes.inputs[input_names::Dropout_scale_inv]) { + last_output = pointwise(last_output, + attributes.inputs[input_names::Dropout_scale_inv], + Pointwise_attributes().set_name("scale_dropout_inv").set_mode(PointwiseMode_t::MUL)); + } else if (has_sink_token) { + // Add identity MUL (multiply by 1) to create MUL→SUB pattern for pattern matcher + auto one_tensor = std::make_shared<Tensor_attributes>(1.0f); + last_output = pointwise(last_output, + one_tensor, + Pointwise_attributes().set_name("identity_for_sink").set_mode(PointwiseMode_t::MUL)); + last_output->set_dim({b, h_q, s_q, 1}).set_stride({h_q * s_q, s_q, 1, 1}); + } + auto softmax_sum = last_output; + + //// Q * K + auto bmm_Q_K_attributes = Matmul_attributes().set_name("bmm_Q_K") + .set_m_override(attributes.inputs[input_names::SEQ_LEN_Q]) + .set_n_override(attributes.inputs[input_names::SEQ_LEN_KV]); + auto last_dV = matmul(eff_Q, eff_K, bmm_Q_K_attributes); + + //// Optional Attn scale + // In case user provided a scalar value, do a fused scalar. + if (attributes.attn_scale_value.has_value()) { + attributes.inputs[input_names::Attn_scale] = + std::make_shared<Tensor_attributes>(attributes.attn_scale_value.value()); + } + + // If attn scale present, add a pointwise mul node + if (auto attn_scale_it = attributes.inputs.find(input_names::Attn_scale); attn_scale_it != attributes.inputs.end()) { + mul_attributes.set_name("attn_scale"); + last_dV = pointwise(last_dV, attn_scale_it->second, mul_attributes); + } + + // Descale Q (only for non-MXFP8 per-tensor scaling) + if (!use_mxfp8) { + mul_attributes.set_name("descale_q"); + last_dV = pointwise(last_dV, attributes.inputs.at(input_names::Descale_Q), mul_attributes); + } + + // Descale K (only for non-MXFP8 per-tensor scaling) + if (!use_mxfp8) { + mul_attributes.set_name("descale_k"); + last_dV = pointwise(last_dV, attributes.inputs.at(input_names::Descale_K), mul_attributes); + } + + // (optional) last_dV = last_dV + bias + if (auto bias_it = attributes.inputs.find(input_names::Bias); bias_it != attributes.inputs.end()) { + last_dV = pointwise(last_dV, + bias_it->second, + Pointwise_attributes().set_name("add_bias").set_mode(PointwiseMode_t::ADD)); + } + + // (optional) Apply padding mask + if (attributes.padding_mask) { + auto row_idx_output = pointwise(last_dV, + Pointwise_attributes() + .set_name("gen_row_idx_padding") + .set_mode(PointwiseMode_t::GEN_INDEX) + .set_axis(2) + .set_compute_data_type(DataType_t::INT32)); + row_idx_output->set_data_type(DataType_t::INT32); + + auto col_idx_output = pointwise(last_dV, + Pointwise_attributes() + .set_name("gen_col_idx_padding") + .set_mode(PointwiseMode_t::GEN_INDEX) + .set_axis(3) + .set_compute_data_type(DataType_t::INT32)); + col_idx_output->set_data_type(DataType_t::INT32); + + auto row_mask_output = pointwise(row_idx_output, + attributes.inputs[input_names::SEQ_LEN_Q], + Pointwise_attributes() + .set_name("lt_row_sq_padding") + .set_mode(PointwiseMode_t::CMP_LT) + .set_compute_data_type(DataType_t::BOOLEAN)); + row_mask_output->set_data_type(DataType_t::BOOLEAN); + + auto col_mask_output = pointwise(col_idx_output, + attributes.inputs[input_names::SEQ_LEN_KV], + Pointwise_attributes() + .set_name("lt_col_skv_padding") + .set_mode(PointwiseMode_t::CMP_LT) + .set_compute_data_type(DataType_t::BOOLEAN)); + col_mask_output->set_data_type(DataType_t::BOOLEAN); + + auto padding_mask_output = pointwise(row_mask_output, + col_mask_output, + Pointwise_attributes() + .set_name("and_row_col_padding") + .set_mode(PointwiseMode_t::LOGICAL_AND) + .set_compute_data_type(DataType_t::BOOLEAN)); + padding_mask_output->set_data_type(DataType_t::BOOLEAN); + + // Use a smaller value of neg infinity so that the softmax stats for rows that are fully padded dont + // go towards NaNs/Infs when multipled by the numerous scale/descale + auto negative_inf_padding = std::make_shared<Tensor_attributes>(attn::score_modifiers::get_negative_inf_value()); + + last_dV = + pointwise(last_dV, + negative_inf_padding, + padding_mask_output, + Pointwise_attributes().set_name("select_padding").set_mode(PointwiseMode_t::BINARY_SELECT)); + } + + // Apply (bottom-right) causal masking (with right bound) and/or set the left bound + if (attributes.left_bound.has_value() || attributes.right_bound.has_value()) { + auto graph_ = std::make_shared<Graph>(); + std::shared_ptr<INode> node_ = std::static_pointer_cast<INode>(graph_); + node_->context = this->context; + + auto s_kv_ptr = attributes.inputs.find(input_names::SEQ_LEN_KV) != attributes.inputs.end() + ? attributes.inputs[input_names::SEQ_LEN_KV] + : nullptr; + auto s_q_ptr = attributes.inputs.find(input_names::SEQ_LEN_Q) != attributes.inputs.end() + ? attributes.inputs[input_names::SEQ_LEN_Q] + : nullptr; + + last_dV = attn::score_modifiers::sliding_window_mask(graph_, + last_dV, + attributes.diagonal_alignment, + attributes.left_bound, + attributes.right_bound, + s_q, + s_kv, + s_q_ptr, + s_kv_ptr, + /*cu_s_q_ptr=*/nullptr, + /*cu_s_kv_ptr=*/nullptr); + sub_nodes.emplace_back(std::move(node_)); + } + + //// Apply Softmax + // last_dV = last_dV - stats + last_dV = pointwise(last_dV, + attributes.inputs[input_names::Stats], + Pointwise_attributes().set_name("sub_dV_Stats").set_mode(PointwiseMode_t::SUB)); + + // last_dV = exp(last_dV) + last_dV = pointwise(last_dV, Pointwise_attributes().set_name("exp_dV").set_mode(PointwiseMode_t::EXP)); + auto exp_S = last_dV; + + // (optional) last_dV = last_dV * dropout rng_output + if (is_dropout_prob || is_dropout_mask) { + last_dV = + pointwise(last_dV, + rng_output, + Pointwise_attributes().set_name("mul_p_dropout_mask").set_mode(PointwiseMode_t::MUL)); + } + + // (optional) last_dV = last_dV * dropout_scale + if (attributes.inputs[input_names::Dropout_scale]) { + last_dV = + pointwise(last_dV, + attributes.inputs[input_names::Dropout_scale], + Pointwise_attributes().set_name("mul_dS_dropout_scale").set_mode(PointwiseMode_t::MUL)); + } + + // Scale S (only for non-MXFP8 per-tensor scaling; MXFP8 keeps P in fp32) + if (!use_mxfp8) { + mul_attributes.set_name("scale_S"); + last_dV = pointwise(last_dV, attributes.inputs.at(input_names::Scale_S), mul_attributes); + last_dV->set_data_type(attributes.inputs.at(input_names::Q)->get_data_type()); + } + + // Reshape P + last_dV = reshape(last_dV, Reshape_attributes().set_name("S_transpose")); + last_dV->set_name("S_T").set_dim({b, h_q, s_kv, s_q}).set_stride({h_q * s_q * s_kv, s_q * s_kv, 1, s_kv}); + if (!use_mxfp8) { + last_dV->set_data_type(attributes.inputs[input_names::Q]->get_data_type()); + } + + //// P.T @ dO -> dV + if (use_mxfp8) { + // MXFP8: Dequantize dO_T with Descale_dO_T for P.T@dO + auto dequant_dO_seq_attrs = Block_scale_dequantize_attributes() + .set_name("DQ_dO_seq") + .set_block_size({1, 32}); + auto dO_seq_dequant = std::make_shared<Tensor_attributes>(); + dO_seq_dequant->set_is_virtual(true); + dO_seq_dequant->set_dim(attributes.inputs[input_names::dO_T]->get_dim()); + dO_seq_dequant->set_stride(attributes.inputs[input_names::dO_T]->get_stride()); + block_scale_dequantize(attributes.inputs[input_names::dO_T], + attributes.inputs[input_names::Descale_dO_T], + dequant_dO_seq_attrs, + dO_seq_dequant); + + auto bmm_S_T_dO_attributes = Matmul_attributes().set_name("bmm_S_T_dO") + .set_m_override(attributes.inputs[input_names::SEQ_LEN_KV]) + .set_k_override(attributes.inputs[input_names::SEQ_LEN_Q]); + if (h_q == h_v) { + matmul(last_dV, dO_seq_dequant, bmm_S_T_dO_attributes, attributes.outputs[output_names::dV]); + } else { + auto dV_fullhead = matmul(last_dV, dO_seq_dequant, bmm_S_T_dO_attributes); + dV_fullhead->set_dim({b, h_q, s_kv, d_v}); + dV_fullhead->set_stride({h_q * s_kv * d_v, s_kv * d_v, d_v, 1}); + dV_fullhead->set_data_type(DataType_t::FLOAT); + reduction(dV_fullhead, + Reduction_attributes().set_name("red_dV_head").set_mode(ReductionMode_t::ADD), + attributes.outputs[output_names::dV]); + } + + auto const& amax_dv = attributes.outputs.at(output_names::Amax_dV); + if (amax_dv != nullptr) { + auto amax_dv_attributes = + Reduction_attributes().set_name("amax_dV").set_mode(ReductionMode_t::AMAX); + reduction(attributes.outputs[output_names::dV], amax_dv_attributes, amax_dv); + } + } else { + matmul_fp8(last_dV, + attributes.inputs[input_names::dO], + attributes.inputs[input_names::Descale_S], + attributes.inputs[input_names::Descale_dO], + attributes.inputs[input_names::Scale_dV], + Matmul_fp8_attributes().set_name("bmm_S_T_dO") + .set_m_override(attributes.inputs[input_names::SEQ_LEN_KV]) + .set_k_override(attributes.inputs[input_names::SEQ_LEN_Q]), + attributes.outputs[output_names::dV], + attributes.outputs[output_names::Amax_dV]); + } + + //// dO * V_T + auto bmm_dO_V_T_attributes = Matmul_attributes().set_name("bmm_dO_V_T") + .set_m_override(attributes.inputs[input_names::SEQ_LEN_Q]) + .set_n_override(attributes.inputs[input_names::SEQ_LEN_KV]); + last_output = matmul(eff_dO, eff_V, bmm_dO_V_T_attributes); + + // Descale dO (only for non-MXFP8 per-tensor scaling) + if (!use_mxfp8) { + mul_attributes.set_name("descale_dO"); + last_output = pointwise(last_output, attributes.inputs.at(input_names::Descale_dO), mul_attributes); + } + + // Descale V (only for non-MXFP8 per-tensor scaling) + if (!use_mxfp8) { + mul_attributes.set_name("descale_V"); + last_output = pointwise(last_output, attributes.inputs.at(input_names::Descale_V), mul_attributes); + } + + // dP = last_output - softmax_sum + auto dP = pointwise(last_output, + softmax_sum, + Pointwise_attributes().set_name("sub_dP_softmax_sum").set_mode(PointwiseMode_t::SUB)); + + // dP = dP * exp_S + mul_attributes.set_name("mul_dP_exp_S"); + dP = pointwise(dP, exp_S, mul_attributes); + + // (optional) dP = dP * dropout_scale + if (attributes.inputs[input_names::Dropout_scale]) { + dP = + pointwise(dP, + attributes.inputs[input_names::Dropout_scale], + Pointwise_attributes().set_name("mul_dS_dropout_scale").set_mode(PointwiseMode_t::MUL)); + } + + // if (attributes.outputs[output_names::dBias]) { + // reduction(dP, + // Reduction_attributes().set_name("red_dP_dBias").set_mode(ReductionMode_t::ADD), + // attributes.outputs[output_names::dBias]); + // } + + // (optional) dP = dP * attn_scale + if (auto attn_scale_it = attributes.inputs.find(input_names::Attn_scale); attn_scale_it != attributes.inputs.end()) { + mul_attributes.set_name("mul_dS_attn_scale"); + dP = pointwise(dP, attn_scale_it->second, mul_attributes); + } + + // Amax dP and Scale dP (only for non-MXFP8 per-tensor scaling; MXFP8 keeps dS in fp32) + if (!use_mxfp8) { + auto amax_attributes = Reduction_attributes().set_name("amax_dP").set_mode(ReductionMode_t::AMAX); + reduction(dP, amax_attributes, attributes.outputs.at(output_names::Amax_dP)); + + mul_attributes.set_name("scale_dP"); + dP = pointwise(dP, attributes.inputs.at(input_names::Scale_dP), mul_attributes); + dP->set_data_type(attributes.inputs.at(input_names::dO)->get_data_type()); + } + + //// dS @ K -> dQ + if (use_mxfp8) { + // MXFP8: Dequantize K_T with Descale_K_T (seq-dimension scaling) for dS@K -> dQ + auto dequant_K_seq_attrs = Block_scale_dequantize_attributes() + .set_name("DQ_K_seq") + .set_block_size({1, 32}); + auto K_seq_dequant = std::make_shared<Tensor_attributes>(); + K_seq_dequant->set_is_virtual(true); + K_seq_dequant->set_dim(attributes.inputs[input_names::K_T]->get_dim()); + K_seq_dequant->set_stride(attributes.inputs[input_names::K_T]->get_stride()); + + int64_t s_kv_scale_padded = ((s_kv + 127) / 128) * 4; + auto& sf_k_T = attributes.inputs[input_names::Descale_K_T]; + sf_k_T->set_stride({h_k * s_kv_scale_padded * 128, s_kv_scale_padded * 128, b * h_k * s_kv_scale_padded * 128, 1}); + + block_scale_dequantize(attributes.inputs[input_names::K_T], + attributes.inputs[input_names::Descale_K_T], + dequant_K_seq_attrs, + K_seq_dequant); + + auto bmm_dS_K_attributes = Matmul_attributes().set_name("bmm_dS_K") + .set_m_override(attributes.inputs[input_names::SEQ_LEN_Q]) + .set_k_override(attributes.inputs[input_names::SEQ_LEN_KV]); + matmul(dP, K_seq_dequant, bmm_dS_K_attributes, attributes.outputs[output_names::dQ]); + + auto const& amax_dq = attributes.outputs.at(output_names::Amax_dQ); + if (amax_dq != nullptr) { + auto amax_dq_attributes = + Reduction_attributes().set_name("amax_dQ").set_mode(ReductionMode_t::AMAX); + reduction(attributes.outputs[output_names::dQ], amax_dq_attributes, amax_dq); + } + } else { + auto const& kt_dim = attributes.inputs[input_names::K]->get_dim(); + auto const& kt_stride = attributes.inputs[input_names::K]->get_stride(); + + auto K = reshape(attributes.inputs[input_names::K], Reshape_attributes().set_name("reshape_K")); + K->set_dim({kt_dim[0], kt_dim[1], kt_dim[3], kt_dim[2]}) + .set_stride({kt_stride[0], kt_stride[1], kt_stride[3], kt_stride[2]}); + + auto bmm_dP_K_attributes = Matmul_fp8_attributes().set_name("bmm_dP_K") + .set_m_override(attributes.inputs[input_names::SEQ_LEN_Q]) + .set_k_override(attributes.inputs[input_names::SEQ_LEN_KV]); + matmul_fp8(dP, + K, + attributes.inputs[input_names::Descale_dP], + attributes.inputs[input_names::Descale_K], + attributes.inputs[input_names::Scale_dQ], + bmm_dP_K_attributes, + attributes.outputs[output_names::dQ], + attributes.outputs[output_names::Amax_dQ]); + } + + //// dS.T * Q (transpose dS -> dS_reshape) + auto dP_T_attributes = Reshape_attributes().set_name("dP_T"); + auto dP_T = reshape(dP, dP_T_attributes); + if (!use_mxfp8) { + dP_T->set_data_type(attributes.inputs.at(input_names::dO)->get_data_type()); + } + dP_T->set_name("dP_T").set_dim({b, h_q, s_kv, s_q}).set_stride({h_q * s_q * s_kv, s_q * s_kv, 1, s_kv}); + + if (use_mxfp8) { + // MXFP8: Dequantize Q_T with Descale_Q_T (seq-dimension scaling) for dS.T@Q -> dK + auto dequant_Q_seq_attrs = Block_scale_dequantize_attributes() + .set_name("DQ_Q_seq") + .set_block_size({1, 32}); + auto Q_seq_dequant = std::make_shared<Tensor_attributes>(); + Q_seq_dequant->set_is_virtual(true); + Q_seq_dequant->set_dim(attributes.inputs[input_names::Q_T]->get_dim()); + Q_seq_dequant->set_stride(attributes.inputs[input_names::Q_T]->get_stride()); + + int64_t s_q_scale_padded = ((s_q + 127) / 128) * 4; + auto& sf_q_T = attributes.inputs[input_names::Descale_Q_T]; + sf_q_T->set_stride({h_q * s_q_scale_padded * 128, s_q_scale_padded * 128, b * h_q * s_q_scale_padded * 128, 1}); + + block_scale_dequantize(attributes.inputs[input_names::Q_T], + attributes.inputs[input_names::Descale_Q_T], + dequant_Q_seq_attrs, + Q_seq_dequant); + + auto bmm_dS_T_Q_attributes = Matmul_attributes().set_name("bmm_dS_T_Q") + .set_m_override(attributes.inputs[input_names::SEQ_LEN_KV]) + .set_k_override(attributes.inputs[input_names::SEQ_LEN_Q]); + if (h_q == h_k) { + matmul(dP_T, Q_seq_dequant, bmm_dS_T_Q_attributes, attributes.outputs[output_names::dK]); + } else { + auto dK_fullhead = matmul(dP_T, Q_seq_dequant, bmm_dS_T_Q_attributes); + dK_fullhead->set_dim({b, h_q, s_kv, d_qk}); + dK_fullhead->set_stride({h_q * s_kv * d_qk, s_kv * d_qk, d_qk, 1}); + dK_fullhead->set_data_type(DataType_t::FLOAT); + reduction(dK_fullhead, + Reduction_attributes().set_name("red_dK_head").set_mode(ReductionMode_t::ADD), + attributes.outputs[output_names::dK]); + } + + auto const& amax_dk = attributes.outputs.at(output_names::Amax_dK); + if (amax_dk != nullptr) { + auto amax_dk_attributes = + Reduction_attributes().set_name("amax_dK").set_mode(ReductionMode_t::AMAX); + reduction(attributes.outputs[output_names::dK], amax_dk_attributes, amax_dk); + } + } else { + auto bmm_dP_T_Q_attributes = Matmul_fp8_attributes().set_name("bmm_dP_T_Q") + .set_m_override(attributes.inputs[input_names::SEQ_LEN_KV]) + .set_k_override(attributes.inputs[input_names::SEQ_LEN_Q]); + matmul_fp8(dP_T, + attributes.inputs[input_names::Q], + attributes.inputs[input_names::Descale_dP], + attributes.inputs[input_names::Descale_Q], + attributes.inputs[input_names::Scale_dK], + bmm_dP_T_Q_attributes, + attributes.outputs[output_names::dK], + attributes.outputs[output_names::Amax_dK]); + } + + return {error_code_t::OK, ""}; + } + + std::pair<int64_t, std::unordered_map<KnobType_t, int64_t>> + override_heuristics_query() const { + int32_t const sm_version = context.get_sm_version(); + bool const use_new_knobs = detail::get_backend_version() >= 92300; + // {128,128} bprop: tileM=3, tileN=2, kernelCfg=2(bprop warp), streamK=0, cgaM=0 + if (sm_version > 103 && is_deterministic_algorithm_supported_on_blackwell) { + if (use_new_knobs) { + return {17, + {{KnobType_t::TILE_M, 3}, + {KnobType_t::TILE_N, 2}, + {KnobType_t::KERNEL_CFG, 2}, + {KnobType_t::STREAM_K, 0}, + {KnobType_t::TILE_CGA_M, 0}, + {KnobType_t::STAGES, 2}}}; + } else { + return {17, {{KnobType_t::KERNEL_CFG, 31}, {KnobType_t::STAGES, 2}}}; + } + } else if (is_deterministic_algorithm_supported_on_blackwell) { + if (use_new_knobs) { + return {5, + {{KnobType_t::TILE_M, 3}, + {KnobType_t::TILE_N, 2}, + {KnobType_t::KERNEL_CFG, 2}, + {KnobType_t::STREAM_K, 0}, + {KnobType_t::TILE_CGA_M, 0}, + {KnobType_t::STAGES, 2}}}; + } else { + return {5, {{KnobType_t::KERNEL_CFG, 31}, {KnobType_t::STAGES, 2}}}; + } + } else { + return {-1, {}}; + } + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j["is_mxfp8"] = is_mxfp8_scaling(); + if (auto const rescale_threshold = get_rescale_threshold_from_env(); rescale_threshold.has_value()) { + j["rescale_threshold"] = rescale_threshold.value(); + } + if (is_mxfp8_scaling()) { + j.update(R"({"tag": "SDPA_MXFP8_BWD"})"_json); + } else { + j.update(R"({"tag": "SDPA_FP8_BWD"})"_json); + } + } +#endif +}; + +} // namespace cudnn_frontend::graph diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/sdpa_support_surface.h b/deps/cudnn-frontend/include/cudnn_frontend/node/sdpa_support_surface.h new file mode 100644 index 0000000..6b73f73 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/sdpa_support_surface.h @@ -0,0 +1,553 @@ +#pragma once + +#include <cstdlib> +#include <unordered_set> + +#include "../../cudnn_frontend_Heuristics.h" +#include "../../cudnn_frontend_Logging.h" +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend::graph { + +inline error_t +SDPA_attributes::validate_sdpa_support_surface(const detail::Context& context, + int64_t s_kv, + bool is_paged_k, + bool is_paged_v) const { + // Extract dimensions from tensors + int64_t s_q = inputs.at(SDPA_attributes::input_names::Q)->get_dim()[2]; + // s_kv is passed in from the caller + int64_t h_q = inputs.at(SDPA_attributes::input_names::Q)->get_dim()[1]; + int64_t h_k = inputs.at(SDPA_attributes::input_names::K)->get_dim()[1]; + int64_t h_v = inputs.at(SDPA_attributes::input_names::V)->get_dim()[1]; + int64_t d_qk = inputs.at(SDPA_attributes::input_names::Q)->get_dim()[3]; + int64_t d_v = inputs.at(SDPA_attributes::input_names::V)->get_dim()[3]; + + bool const is_ragged = inputs.at(SDPA_attributes::input_names::Q)->get_ragged_offset() || + inputs.at(SDPA_attributes::input_names::K)->get_ragged_offset() || + inputs.at(SDPA_attributes::input_names::V)->get_ragged_offset() || + outputs.at(SDPA_attributes::output_names::O)->get_ragged_offset(); + + auto const& output_tensor = outputs.at(SDPA_attributes::output_names::O); + auto const& output_data_type = output_tensor->get_data_type(); + + auto const& bias_mask = inputs.find(SDPA_attributes::input_names::Bias); + bool const is_bias = (bias_mask != inputs.end() && bias_mask->second != nullptr); + + auto const& dropout_mask = inputs.find(SDPA_attributes::input_names::Dropout_mask); + bool const is_dropout_custom = (dropout_mask != inputs.end()) && (dropout_mask->second != nullptr); + bool const is_dropout = dropout_probability.has_value() || is_dropout_custom; + + bool const is_paged = is_paged_k || is_paged_v; + + auto const& rng_tensor = outputs.find(SDPA_attributes::output_names::RNG_DUMP); + bool const is_rng = (rng_tensor != outputs.end() && rng_tensor->second != nullptr); + + bool const max_seq_kv_explicit = max_seq_len_kv.has_value(); + + auto const& attn_scale = inputs.find(SDPA_attributes::input_names::Attn_scale); + bool const has_attn_scale = (attn_scale != inputs.end()) && (attn_scale->second != nullptr); + + auto const& seq_len_q = inputs.find(SDPA_attributes::input_names::SEQ_LEN_Q); + bool const has_seq_len_q = (seq_len_q != inputs.end()) && (seq_len_q->second != nullptr); + auto const& seq_len_kv = inputs.find(SDPA_attributes::input_names::SEQ_LEN_KV); + bool const has_seq_len_kv = (seq_len_kv != inputs.end()) && (seq_len_kv->second != nullptr); + + auto const& cu_seq_len_q = inputs.find(SDPA_attributes::input_names::CU_SEQ_LEN_Q); + bool const has_cu_seq_len_q = (cu_seq_len_q != inputs.end()) && (cu_seq_len_q->second != nullptr); + auto const& cu_seq_len_kv = inputs.find(SDPA_attributes::input_names::CU_SEQ_LEN_KV); + bool const has_cu_seq_len_kv = (cu_seq_len_kv != inputs.end()) && (cu_seq_len_kv->second != nullptr); + + // validation TODO: + // - validate stats has valid dims + + // Get device SM version from context + CHECK_CUDNN_FRONTEND_ERROR(context.populate_sm_version_from_device()); + int32_t const sm_version = context.get_sm_version(); + int32_t const prop_major = sm_version / 10; + + // Common FP16 and FP8 validation + // validate basic dimension requirements + RETURN_CUDNN_FRONTEND_ERROR_IF( + (h_q % h_k != 0) || (h_q % h_v != 0), + error_code_t::GRAPH_NOT_SUPPORTED, + "For group-query attention, number of heads for key and query must be a factor of number of heads for query"); + + // validate options for attn_scale + RETURN_CUDNN_FRONTEND_ERROR_IF(has_attn_scale && attn_scale_value.has_value(), + error_code_t::ATTRIBUTE_NOT_SET, + "attn_scale with tensor and value cannot be set at the same time."); + + // validate options for bias mask + RETURN_CUDNN_FRONTEND_ERROR_IF(is_bias && (bias_mask->second->get_data_type() == DataType_t::BOOLEAN), + error_code_t::GRAPH_NOT_SUPPORTED, + "Bias mask data type cannot be boolean"); + + // validate options for padding mask: padding requires per-sequence length tensors, + // either as (SEQ_LEN_Q + SEQ_LEN_KV) or as (CU_SEQ_LEN_Q + CU_SEQ_LEN_KV). + RETURN_CUDNN_FRONTEND_ERROR_IF( + padding_mask && (!has_seq_len_q || !has_seq_len_kv) && (!has_cu_seq_len_q || !has_cu_seq_len_kv), + error_code_t::ATTRIBUTE_NOT_SET, + "Padding mask requires seq_len_q/seq_len_kv (or cu_seq_len_q/cu_seq_len_kv) to be set."); + RETURN_CUDNN_FRONTEND_ERROR_IF( + (!padding_mask && !attention_score_modifier) && + (has_seq_len_q || has_seq_len_kv || has_cu_seq_len_q || has_cu_seq_len_kv), + error_code_t::ATTRIBUTE_NOT_SET, + "seq_len_q/seq_len_kv (or cu_seq_len_q/cu_seq_len_kv) needs to be set only if padding mask is enabled."); + + // Cumulative sequence length tensors must be set together, and are mutually + // exclusive with the (per-batch) seq_len tensors. + RETURN_CUDNN_FRONTEND_ERROR_IF(has_cu_seq_len_q != has_cu_seq_len_kv, + error_code_t::ATTRIBUTE_NOT_SET, + "cu_seq_len_q and cu_seq_len_kv must both be set or both unset."); + RETURN_CUDNN_FRONTEND_ERROR_IF((has_cu_seq_len_q || has_cu_seq_len_kv) && (has_seq_len_q || has_seq_len_kv), + error_code_t::INVALID_VALUE, + "Cannot specify both seq_len tensors and cu_seq_len tensors."); + + RETURN_CUDNN_FRONTEND_ERROR_IF(is_ragged && ((padding_mask == false) && (attention_score_modifier == nullptr)), + error_code_t::GRAPH_NOT_SUPPORTED, + "Ragged offsets are only supported with padding mask."); + + // validate options for dropout mask + RETURN_CUDNN_FRONTEND_ERROR_IF( + dropout_probability.has_value() && is_dropout_custom, + error_code_t::ATTRIBUTE_NOT_SET, + "Using both, custom dropout mask and internal-mask generation using dropout probability, is ill-formed."); + + RETURN_CUDNN_FRONTEND_ERROR_IF(dropout_probability.has_value() && dropout_probability.value() == 1.0, + error_code_t::ATTRIBUTE_NOT_SET, + "Dropout probability cannot be 1 as corresponding scale wont be well formed."); + + // validate options for causal mask and bottom right causal mask + RETURN_CUDNN_FRONTEND_ERROR_IF( + has_causal_mask_bottom_right() && (!padding_mask) && s_q > s_kv, + error_code_t::GRAPH_NOT_SUPPORTED, + "Bottom right causal mask does not support max_s_q > max_s_kv. Please virtually slice the Q tensor and pass it " + "as max_s_q == max_s_kv"); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + has_causal_mask_bottom_right() && (is_bias || alibi_mask || is_dropout), + error_code_t::GRAPH_NOT_SUPPORTED, + "Bottom right causal mask is only supported with is_bias=False, is_alibi=False, is_dropout=False."); + + RETURN_CUDNN_FRONTEND_ERROR_IF(has_causal_mask_bottom_right() && (detail::get_backend_version() < 90600) && + ((s_q % 64 != 0) || (s_kv % 64 != 0)), + error_code_t::GRAPH_NOT_SUPPORTED, + "Bottom right causal mask is only supported with s_q multiple of 64, and s_kv " + "multiple of 64, for cudnn version below 9.6.0"); + + RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 90600 && left_bound.has_value() && + has_causal_mask_bottom_right() && s_q != s_kv, + error_code_t::GRAPH_NOT_SUPPORTED, + "Sliding window attention with bottom right causal mask requires s_q == s_kv " + "for cuDNN version below 9.6.0"); + + // validate that datatype is set for the graph + RETURN_CUDNN_FRONTEND_ERROR_IF(context.get_intermediate_data_type() == DataType_t::NOT_SET, + error_code_t::ATTRIBUTE_NOT_SET, + "Intermediate tensor data type needs to be set as internal tensors require it."); + + if (mma_core_mode == DataType_t::FP8_E4M3 || mma_core_mode == DataType_t::FP8_E5M2) { + // FP8 specific validation + + RETURN_CUDNN_FRONTEND_ERROR_IF((prop_major == 12) && is_ragged, + error_code_t::GRAPH_NOT_SUPPORTED, + "sdpa fp8 with THD not supported for sm120 yet."); + + // version specific validation + RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 90100, + error_code_t::GRAPH_NOT_SUPPORTED, + "sdpa fp8 forward operation is only supported starting cudnn 9.1.0. Please " + "consider upgrading your current version."); + RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() == 91000, + error_code_t::GRAPH_NOT_SUPPORTED, + "sdpa fp8 forward operation is not supported on cudnn 9.10.0. Please " + "consider upgrading your current version."); + RETURN_CUDNN_FRONTEND_ERROR_IF( + prop_major < 9, + error_code_t::GRAPH_NOT_SUPPORTED, + "sdpa fp8 forward operation is only supported on Hopper architecture and newer. Please " + "consider using a newer architecture."); + + // FP8 does not support bias (TE constraint) + RETURN_CUDNN_FRONTEND_ERROR_IF(is_bias, error_code_t::GRAPH_NOT_SUPPORTED, "SDPA FP8 does not support bias"); + + // validate basic dimension requirements + // d_qk=192 with d_v=128 is only supported starting from cuDNN 9.19 + bool const d192_v128_supported = (detail::get_backend_version() >= 91900); + if (prop_major >= 10) { + RETURN_CUDNN_FRONTEND_ERROR_IF( + ((d_qk > 128) || (d_qk % 16 != 0)) && !(d192_v128_supported && d_qk == 192 && d_v == 128), + error_code_t::GRAPH_NOT_SUPPORTED, + "hidden_dim d_qk should be less than or equal to 128 and hidden_dim d_qk " + "should be multiple of 16 unless d_qk == 192 and d_v == 128 (requires cuDNN 9.19+)"); + RETURN_CUDNN_FRONTEND_ERROR_IF( + ((d_v > 128) || (d_v % 16 != 0)), + error_code_t::GRAPH_NOT_SUPPORTED, + "hidden_dim d_v should be less than or equal to 128 and hidden_dim d_v should be multiple of 16"); + } else { + RETURN_CUDNN_FRONTEND_ERROR_IF( + (d_qk > 256) || (d_qk % 16 != 0) || (d_v > 256) || (d_v % 16 != 0), + error_code_t::GRAPH_NOT_SUPPORTED, + "hidden_dim shoud be less than or equal to 256 and hidden_dim should be multiple of 16"); + } + + // Validate options for causal_mask_bottom_right + RETURN_CUDNN_FRONTEND_ERROR_IF(has_causal_mask_bottom_right() && detail::get_backend_version() < 90700, + error_code_t::GRAPH_NOT_SUPPORTED, + "For cuDNN version below 9.7.0, bottom right causal masking is not supported."); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + has_causal_mask_bottom_right() && prop_major < 10, + error_code_t::GRAPH_NOT_SUPPORTED, + "sdpa fp8 forward operation is only supported on Blackwell architecture and newer. Please " + "consider using a newer architecture."); + + // if output data type is half or bfloat16, and version is below 9.13 or is not blackwell, return NOT_SUPPORTED + RETURN_CUDNN_FRONTEND_ERROR_IF( + (output_data_type == DataType_t::HALF || output_data_type == DataType_t::BFLOAT16) && + (detail::get_backend_version() < 91300 || prop_major < 10), + error_code_t::GRAPH_NOT_SUPPORTED, + "sdpa fp8 forward with HALF/BFLOAT16 output is only supported on Blackwell architecture " + "with cuDNN version 9.13.0 and newer."); + } else if (mma_core_mode == DataType_t::HALF) { + // FP16 specific validation + + // Bug workarounds for known problematic versions + RETURN_CUDNN_FRONTEND_ERROR_IF( + detail::get_backend_version() == 91000 || detail::get_backend_version() == 91001, + error_code_t::GRAPH_NOT_SUPPORTED, + "SDPA FP16/BF16 forward is not supported on cuDNN 9.10.0/9.10.1 due to known bugs. " + "Please consider upgrading to 9.10.2 or newer."); + + // 9.14.0 sliding window bug: non-causal + s_kv > 1024 + sliding window + RETURN_CUDNN_FRONTEND_ERROR_IF( + detail::get_backend_version() == 91400 && s_kv > 1024 && left_bound.has_value() && + !has_causal_like_masking(), + error_code_t::GRAPH_NOT_SUPPORTED, + "cuDNN 9.14.0 has a known bug with non-causal + s_kv > 1024 + sliding window attention. " + "Please consider upgrading to 9.14.1 or newer."); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + (attention_score_modifier != nullptr) && + (alibi_mask || has_causal_like_masking() || padding_mask || left_bound.has_value()), + error_code_t::GRAPH_NOT_SUPPORTED, + "Attention score mod enabled and hence other subgraphs are disabled."); + + // validate basic dimension requirements + RETURN_CUDNN_FRONTEND_ERROR_IF( + (d_qk % 8 != 0) || (d_v % 8 != 0), error_code_t::GRAPH_NOT_SUPPORTED, "hidden_dim should be multiple of 8"); + + // validate alibi requirements + RETURN_CUDNN_FRONTEND_ERROR_IF(alibi_mask && !(right_bound.has_value() && right_bound.value() == 0), + error_code_t::GRAPH_NOT_SUPPORTED, + "When alibi mask is used, diagonal_band_right_bound needs to be set to 0."); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + left_bound.has_value() && left_bound.value() <= 0 && detail::get_backend_version() < 91000, + error_code_t::INVALID_VALUE, + "Left bound (Sliding window length) should be greater than zero when set."); + + RETURN_CUDNN_FRONTEND_ERROR_IF(left_bound.has_value() && (!padding_mask) && s_q > s_kv, + error_code_t::GRAPH_NOT_SUPPORTED, + "Sliding window attention is only supported with max_s_q <= max_s_kv."); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + left_bound.has_value() && (s_q * left_bound.value() == s_kv * left_bound.value()) && + (detail::get_backend_version() <= 90900) && (prop_major == 9) && has_causal_mask_bottom_right(), + error_code_t::GRAPH_NOT_SUPPORTED, + "On Hopper architecture, this specific combination of s_q, s_kv, and left_bound + right_bound + bottom " + "right diagonal alignment is not supported for backend version 9.9 or below"); + + if ((detail::get_backend_version() < 91002)) { + RETURN_CUDNN_FRONTEND_ERROR_IF( + left_bound.has_value() && (!has_causal_like_masking() || is_dropout || is_bias), + error_code_t::GRAPH_NOT_SUPPORTED, + "Left and right bounds are only supported with is_dropout=False, is_bias=False. And the diagonal " + "alignment must be set."); + } + + RETURN_CUDNN_FRONTEND_ERROR_IF(right_bound.has_value() && right_bound.value() < 0, + error_code_t::INVALID_VALUE, + "Right bound needs to be larger than or equal to zero"); + + // Validate options for s_q == 1 + const bool is_decode_only = (s_q == 1); + RETURN_CUDNN_FRONTEND_ERROR_IF(is_decode_only && (prop_major == 10) && (d_qk > 128 || d_v > 128) && + (detail::get_backend_version() <= 90900), + error_code_t::GRAPH_NOT_SUPPORTED, + "decode only mode, i.e. s_q == 1 not supported for blackwell architecture with " + "d_qk or d_v > 128 for backend version 9.9 or below"); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + is_decode_only && (detail::get_backend_version() <= 90900) && (right_bound.has_value()), + error_code_t::GRAPH_NOT_SUPPORTED, + "decode only mode, i.e. s_q == 1, not supported with masking (right_bound is set) for backend version 9.9 " + "or below"); + + RETURN_CUDNN_FRONTEND_ERROR_IF(is_decode_only && has_sink_token(), + error_code_t::GRAPH_NOT_SUPPORTED, + "decode only mode, i.e. s_q == 1, not supported with sink_token"); + + // validate options for paged attention + RETURN_CUDNN_FRONTEND_ERROR_IF( + is_paged && (d_qk > 128 || d_v > 128) && detail::get_backend_version() <= 90900, + error_code_t::GRAPH_NOT_SUPPORTED, + "Paged attention only supported with d_qk and d_v <= 128 for backend version 9.9 or below"); + + RETURN_CUDNN_FRONTEND_ERROR_IF(is_paged && is_ragged && detail::get_backend_version() < 90700, + error_code_t::GRAPH_NOT_SUPPORTED, + "Paged caches are not supported in combination with ragged offsets."); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + is_paged && !((has_seq_len_q && has_seq_len_kv) || (has_cu_seq_len_q && has_cu_seq_len_kv)), + error_code_t::GRAPH_NOT_SUPPORTED, + "Paged caches can only be used in combination with padding mask and variable sequence lengths " + "for both Q and KV (via seq_len_q/seq_len_kv or cu_seq_len_q/cu_seq_len_kv)."); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + !is_paged && max_seq_kv_explicit, + error_code_t::GRAPH_NOT_SUPPORTED, + "When not using paged attention, there is no need to explicitly set max kv sequence length."); + + if (max_seq_kv_explicit) { + auto max_seq_kv = max_seq_len_kv.value(); + + RETURN_CUDNN_FRONTEND_ERROR_IF(is_bias && (bias_mask->second->get_dim()[3] != max_seq_kv), + error_code_t::GRAPH_NOT_SUPPORTED, + "Value set through set_paged_attention_max_seq_len_kv is incompatible with " + "the sequence length of the bias"); + + RETURN_CUDNN_FRONTEND_ERROR_IF(is_rng && rng_tensor->second->get_dim()[3] != max_seq_kv, + error_code_t::GRAPH_NOT_SUPPORTED, + "Value set through set_paged_attention_max_seq_len_kv is incompatible with " + "the sequence length of the RNG_DUMP"); + } + + // Additional validation for paged attention with packed page tables + RETURN_CUDNN_FRONTEND_ERROR_IF( + ((is_paged_k && inputs.at(SDPA_attributes::input_names::Page_table_K)->get_ragged_offset()) || + (is_paged_v && inputs.at(SDPA_attributes::input_names::Page_table_V)->get_ragged_offset())) && + detail::get_backend_version() < 91002, + error_code_t::GRAPH_NOT_SUPPORTED, + "Paged attention with packed page tables only supported with cudnn version 9.10.2 and above"); + + // SM version check for SDPA + RETURN_CUDNN_FRONTEND_ERROR_IF(prop_major < 8, + error_code_t::GRAPH_NOT_SUPPORTED, + "SDPA FP16/BF16 requires SM80 (Ampere) or newer architecture"); + + // version specific validation by architecture + if (prop_major == 8) { + RETURN_CUDNN_FRONTEND_ERROR_IF( + detail::get_backend_version() <= 90900 && ((d_qk > 128) || (d_v > 128)), + error_code_t::GRAPH_NOT_SUPPORTED, + "head_dim should be less than or equal to 128 for backend version 9.9 or below on ampere architecture"); + } + if (prop_major == 9) { + RETURN_CUDNN_FRONTEND_ERROR_IF( + detail::get_backend_version() <= 90900 && ((d_qk > 256) || (d_v > 256)), + error_code_t::GRAPH_NOT_SUPPORTED, + "head_dim should be less than or equal to 256 for backend version 9.9 or below on hopper architecture"); + } + if (prop_major == 10) { + RETURN_CUDNN_FRONTEND_ERROR_IF((detail::get_backend_version() < 90900) && ((d_qk > 128) || (d_v > 128)), + error_code_t::GRAPH_NOT_SUPPORTED, + "head_dim should be less than or equal to 128 for backend version 9.8 or " + "below on blackwell architecture"); + } + + RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 90500 && is_paged, + error_code_t::GRAPH_NOT_SUPPORTED, + "For cuDNN version below 9.5.0, paged caches are not supported"); + + if (is_ragged) { + RETURN_CUDNN_FRONTEND_ERROR_IF((context.get_sm_version() > 0 && context.get_sm_version() < 90 && + detail::get_backend_version() < 91801), + error_code_t::GRAPH_NOT_SUPPORTED, + "THD (ragged offset) is only supported in Hopper and above : " + + std::to_string(context.get_sm_version())); + } + } else { + RETURN_CUDNN_FRONTEND_ERROR_IF(true, error_code_t::GRAPH_NOT_SUPPORTED, "Unsupported mma core mode"); + } + + // Check whether the selected implementation supports the requested features. + CHECK_CUDNN_FRONTEND_ERROR(verify_sdpa_support_surface_for_implementation(context, implementation)); + + return {error_code_t::OK, ""}; +} + +// Verify that the underlying implementation supports all the features in these attributes. +// Unlike `validate_sdpa_support_surface()`, this may be called before validation, so: +// * don't assume any particular keys already exist in `inputs` or `outputs` +// * don't assume any tensor dims or strides are already set +// We return error codes directly instead of using `RETURN_CUDNN_FRONTEND_ERROR_IF` +// to avoid unneeded logging when this function is being called in a non-error-generating +// situation (e.g. during auto-select of SDPA implementation). +inline error_t +SDPA_attributes::verify_sdpa_support_surface_for_implementation(const detail::Context& context, + AttentionImplementation_t impl) const { + switch (impl) { + case AttentionImplementation_t::AUTO: + // This function should not be called with AUTO. + return {error_code_t::INVALID_VALUE, + "Can't call verify_sdpa_support_surface_for_implementation with impl=AUTO"}; + case AttentionImplementation_t::COMPOSITE: + for (const auto& [key, value] : inputs) { + RETURN_CUDNN_FRONTEND_ERROR_IF(key == input_names::Block_mask && value != nullptr, + error_code_t::GRAPH_NOT_SUPPORTED, + "Composite SDPA node doesn't support Block_mask input"); + RETURN_CUDNN_FRONTEND_ERROR_IF( + (key == input_names::CU_SEQ_LEN_Q || key == input_names::CU_SEQ_LEN_KV) && value != nullptr, + error_code_t::GRAPH_NOT_SUPPORTED, + "Composite SDPA node doesn't support CU_SEQ_LEN_Q / CU_SEQ_LEN_KV inputs"); + } + // The ragged offset multiplier is only supported by the unified forward engine. + // Reject it here so auto-select routes such graphs to the unified implementation. + for (const auto& [key, value] : inputs) { + RETURN_CUDNN_FRONTEND_ERROR_IF(value != nullptr && value->has_ragged_offset_multiplier(), + error_code_t::GRAPH_NOT_SUPPORTED, + "Composite SDPA node doesn't support a ragged offset multiplier"); + } + for (const auto& [key, value] : outputs) { + RETURN_CUDNN_FRONTEND_ERROR_IF(value != nullptr && value->has_ragged_offset_multiplier(), + error_code_t::GRAPH_NOT_SUPPORTED, + "Composite SDPA node doesn't support a ragged offset multiplier"); + } + break; + case AttentionImplementation_t::UNIFIED: { + auto effective_cudnn_ver = std::min(detail::get_backend_version(), detail::get_compiled_version()); + RETURN_CUDNN_FRONTEND_ERROR_IF(effective_cudnn_ver < 91301, + error_code_t::GRAPH_NOT_SUPPORTED, + "Unified SDPA node requires cuDNN 9.13.1"); + + RETURN_CUDNN_FRONTEND_ERROR_IF(context.get_dynamic_shape_enabled(), + error_code_t::GRAPH_NOT_SUPPORTED, + "Unified SDPA node doesn't yet support dynamic shape"); + + // TODO: Provide smarter error messages that provide the required cuDNN version for each input. + std::unordered_set<SDPA_attributes::input_names> allowed_input_names{ + input_names::Q, input_names::K, input_names::V, input_names::Attn_scale}; + std::string allowed_input_msg = + "Unified SDPA node doesn't yet support inputs other than Q, K, V, Attn_scale"; + + if (effective_cudnn_ver >= 91400) { + allowed_input_names.insert({input_names::Block_mask}); + allowed_input_msg += ", Block_mask"; + } + + if (effective_cudnn_ver >= 91500) { + allowed_input_names.insert({input_names::Page_table_K, + input_names::Page_table_V, + input_names::SEQ_LEN_Q, + input_names::SEQ_LEN_KV}); + allowed_input_msg += ", Page_table_K, Page_table_V, SEQ_LEN_Q, SEQ_LEN_KV"; + } + + if (effective_cudnn_ver >= 92100) { + // NOTE: For unified engine, we support dropout via Seed and Offset only. + // Custom dropout mask (via Dropout_mask, Dropout_scale) is not supported. + allowed_input_names.insert( + {input_names::Bias, input_names::Seed, input_names::Offset, input_names::SINK_TOKEN}); + allowed_input_msg += ", Bias, Seed, Offset, SINK_TOKEN"; + } + + if (effective_cudnn_ver >= 92400) { + allowed_input_names.insert({input_names::CU_SEQ_LEN_Q, input_names::CU_SEQ_LEN_KV}); + allowed_input_msg += ", CU_SEQ_LEN_Q, CU_SEQ_LEN_KV"; + } + + if (effective_cudnn_ver >= 93000) { + allowed_input_names.insert({input_names::Descale_Q, + input_names::Descale_K, + input_names::Descale_V, + input_names::Descale_S, + input_names::Scale_S, + input_names::Scale_O}); + allowed_input_msg += ", Descale_Q, Descale_K, Descale_V, Descale_S, Scale_S, Scale_O"; + } + + for (const auto& [key, value] : inputs) { + if (allowed_input_names.find(key) == allowed_input_names.end() && value != nullptr) { + return {error_code_t::GRAPH_NOT_SUPPORTED, allowed_input_msg}; + } + } + + std::unordered_set<SDPA_attributes::output_names> allowed_output_names{output_names::O, + output_names::Stats}; + std::string allowed_output_msg = "Unified SDPA node doesn't yet support outputs other than O, Stats"; + + if (effective_cudnn_ver >= 92100) { + allowed_output_names.insert({output_names::RNG_DUMP, output_names::Max, output_names::Sum_exp}); + allowed_output_msg += ", RNG_DUMP, Max, Sum_exp"; + } + + if (effective_cudnn_ver >= 93000) { + allowed_output_names.insert({output_names::Amax_S, output_names::Amax_O}); + allowed_output_msg += ", Amax_S, Amax_O"; + } + + for (const auto& [key, value] : outputs) { + if (allowed_output_names.find(key) == allowed_output_names.end() && value != nullptr) { + return {error_code_t::GRAPH_NOT_SUPPORTED, allowed_output_msg}; + } + } + + if (alibi_mask && effective_cudnn_ver < 92100) { + return {error_code_t::GRAPH_NOT_SUPPORTED, + "Alibi mask for unified SDPA node requires cuDNN 9.21.0 or above"}; + } + + if (padding_mask && effective_cudnn_ver < 91500) { + return {error_code_t::GRAPH_NOT_SUPPORTED, + "Padding mask for unified SDPA node requires cuDNN 9.15.0 or above"}; + } + + if ((left_bound.has_value() || right_bound.has_value()) && effective_cudnn_ver < 92100) { + return {error_code_t::GRAPH_NOT_SUPPORTED, + "Left bound or right bound for unified SDPA node requires cuDNN 9.21.0 or above"}; + } + + if (diagonal_alignment != DiagonalAlignment_t::TOP_LEFT && effective_cudnn_ver < 92100) { + return {error_code_t::GRAPH_NOT_SUPPORTED, + "Diagonal alignment for unified SDPA node requires cuDNN 9.21.0 or above"}; + } + + if (dropout_probability.has_value() && effective_cudnn_ver < 92200) { + return {error_code_t::GRAPH_NOT_SUPPORTED, "Dropout for unified SDPA node requires cuDNN 9.22.0"}; + } + + if (dropout_probability.has_value() && generate_stats.value_or(false)) { + return {error_code_t::GRAPH_NOT_SUPPORTED, + "Dropout for unified SDPA node with generated stats is not supported"}; + } + + // Unified engine in cuDNN < 9.15 can't meaningfully support max sequence length, + // while versions >= 9.15 "support" it by ignoring it (unified engine doesn't need it). + if (max_seq_len_kv.has_value() && effective_cudnn_ver < 91500) { + return {error_code_t::GRAPH_NOT_SUPPORTED, + "Max sequence length for unified SDPA node cannot be set in cuDNN < 9.15.0"}; + } + + if (attention_score_modifier != nullptr && effective_cudnn_ver < 92100) { + return {error_code_t::GRAPH_NOT_SUPPORTED, + "Attention score modifier for unified SDPA node requires cuDNN 9.21.0 or above"}; + } + + if (mma_core_mode == DataType_t::FP8_E4M3 || mma_core_mode == DataType_t::FP8_E5M2) { + // Per-tensor FP8 and MXFP8 (block-scaled, E8M0 descales) are supported by the + // unified node starting from cuDNN 9.30.0. + if (effective_cudnn_ver < 93000) { + return {error_code_t::GRAPH_NOT_SUPPORTED, + "FP8/MXFP8 for the unified SDPA node requires cuDNN 9.30.0 or above"}; + } + } + } break; + } + + return {error_code_t::OK, ""}; +} + +} // namespace cudnn_frontend::graph diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/slice.h b/deps/cudnn-frontend/include/cudnn_frontend/node/slice.h new file mode 100644 index 0000000..a1aafb1 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/slice.h @@ -0,0 +1,224 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend::graph { + +class SliceNode : public NodeCRTP<SliceNode> { + public: + Slice_attributes attributes; + + SliceNode(Slice_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::SLICE; + } + + error_t + infer_properties_node() override final { + getLogger() << "[cudnn_frontend] INFO: Inferrencing properties for slice node " << attributes.name + << std::endl; + + attributes.fill_from_context(context); + + for (size_t i = 0; i < attributes.slice_strides.size(); ++i) { + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.slice_strides[i] <= 0, + error_code_t::INVALID_VALUE, + "Slice slice_strides[" + std::to_string(i) + "] must be strictly positive (got " + + std::to_string(attributes.slice_strides[i]) + + "). Non-positive strides break output dimension calculation and can cause division by zero."); + } + + auto output = attributes.outputs.at(Slice_attributes::output_names::Y); + auto output_dim = output->get_dim(); + + if (output_dim.empty()) { + for (size_t i = 0; i < attributes.slices.size(); ++i) { + int64_t start = attributes.slices[i].first; + int64_t limit = attributes.slices[i].second; + int64_t stride = (!attributes.slice_strides.empty() && i < attributes.slice_strides.size()) + ? attributes.slice_strides[i] + : 1; + // Output dimension = ceil((limit - start) / stride) + int64_t dim = (limit - start + stride - 1) / stride; + output_dim.push_back(dim); + } + output->set_dim(output_dim); + } + + auto const input = attributes.inputs.at(Slice_attributes::input_names::X); + auto const input_data_type = input->get_data_type(); + auto const output_data_type = output->get_data_type(); + if (output_data_type == DataType_t::NOT_SET) { + output->set_data_type(input_data_type); + } else { + RETURN_CUDNN_FRONTEND_ERROR_IF(output_data_type != input_data_type, + error_code_t::INVALID_VALUE, + "output and input tensor data types should match for slice operation."); + } + + auto const input_stride = input->get_stride(); + if (output->get_stride().empty()) { + // When slice strides > 1, output strides need to be multiplied accordingly + std::vector<int64_t> output_stride; + for (size_t i = 0; i < input_stride.size(); ++i) { + int64_t stride = (!attributes.slice_strides.empty() && i < attributes.slice_strides.size()) + ? attributes.slice_strides[i] + : 1; + output_stride.push_back(input_stride[i] * stride); + } + output->set_stride(output_stride); + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_tensors_node(std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors, + int64_t& potential_uid, + std::unordered_set<int64_t> const& used_uids) const override final { + getLogger() << "[cudnn_frontend] INFO: Creating cudnn tensors for SliceNode " << attributes.name << std::endl; + + auto const input = attributes.inputs.at(Slice_attributes::input_names::X); + auto const output = attributes.outputs.at(Slice_attributes::output_names::Y); + + if (detail::get_backend_version() >= 92200 && detail::get_compiled_version() >= 92200) { + CHECK_CUDNN_FRONTEND_ERROR(detail::create_cudnn_tensor(input, tensors, potential_uid, used_uids)); + CHECK_CUDNN_FRONTEND_ERROR(detail::create_cudnn_tensor(output, tensors, potential_uid, used_uids)); + return {error_code_t::OK, ""}; + } + + if (input->has_uid() == false) { + detail::assign_uid(input.get(), potential_uid, used_uids); + } + output->set_is_virtual(false); + CHECK_CUDNN_FRONTEND_ERROR(detail::create_cudnn_tensor(output, tensors, potential_uid, used_uids)); + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations, + std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final { + getLogger() << "[cudnn_frontend] INFO: " << "Building SliceNode operations " << attributes.name << std::endl; + +#if (CUDNN_VERSION >= 92200) + // cuDNN >= 9.22.0: Use native backend slice operation + auto cudnn_ver_error = error_t{error_code_t::GRAPH_NOT_SUPPORTED, "Slice backend operation requires 9.22.0"}; + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(92200, cudnn_ver_error); + + if (detail::get_backend_version() >= 92200) { + CUDNN_FRONTEND_UNUSED(operations); + + auto slice_operation = make_shared_backend_pointer(CUDNN_BACKEND_OPERATION_SLICE_DESCRIPTOR); + + // Set input tensor + auto X = attributes.inputs.at(Slice_attributes::input_names::X); + auto backend_x = tensors[X->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(slice_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SLICE_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_x)); + + // Set output tensor + auto Y = attributes.outputs.at(Slice_attributes::output_names::Y); + auto backend_y = tensors[Y->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(slice_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SLICE_YDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_y)); + + // Extract start and limit indices from slices + std::vector<int64_t> start_indices; + std::vector<int64_t> limit_indices; + + for (const auto& slice : attributes.slices) { + start_indices.push_back(slice.first); + limit_indices.push_back(slice.second); + } + + // Per-dimension strides: use user slice_strides[i] when set, else 1 (preserves partial configuration) + std::vector<int64_t> strides(attributes.slices.size()); + for (size_t i = 0; i < strides.size(); ++i) { + strides[i] = (i < attributes.slice_strides.size()) ? attributes.slice_strides[i] : 1; + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(slice_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SLICE_START_INDICES, + CUDNN_TYPE_INT64, + start_indices.size(), + start_indices.data())); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(slice_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SLICE_LIMIT_INDICES, + CUDNN_TYPE_INT64, + limit_indices.size(), + limit_indices.data())); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(slice_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SLICE_STRIDES, + CUDNN_TYPE_INT64, + strides.size(), + strides.data())); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(slice_operation->get_backend_descriptor())); + + raw_operations.push_back(slice_operation); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } +#endif + CUDNN_FRONTEND_UNUSED(operations); + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FRONTEND_UNUSED(tensors); + + getLogger() << "[cudnn_frontend] INFO: " << "Using pointer arithmetic fallback for slice on cuDNN < 9.22.0" + << std::endl; + + auto const output = attributes.outputs.at(Slice_attributes::output_names::Y); + if (output && output->get_is_virtual() == false) { + uids_involved_in_operations.insert(output->get_uid()); + if (auto ragged_offset = output->get_ragged_offset()) { + uids_involved_in_operations.insert(ragged_offset->get_uid()); + } + } + return {error_code_t::OK, ""}; + } + + error_t + collect_variant_pack_replacements_node( + std::unordered_map<Tensor_attributes::uid_t, std::pair<Tensor_attributes::uid_t, int64_t>>& + variant_pack_replacements) const override final { + if (detail::get_backend_version() >= 92200 && detail::get_compiled_version() >= 92200) { + CUDNN_FRONTEND_UNUSED(variant_pack_replacements); + return {error_code_t::OK, ""}; + } + auto const input = attributes.inputs.at(Slice_attributes::input_names::X); + auto const output = attributes.outputs.at(Slice_attributes::output_names::Y); + + variant_pack_replacements[input->get_uid()] = {output->get_uid(), attributes.get_offset()}; + + return {error_code_t::OK, ""}; + }; + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "SLICE"})"_json); + } +#endif +}; + +} // namespace cudnn_frontend::graph
\ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/softmax.h b/deps/cudnn-frontend/include/cudnn_frontend/node/softmax.h new file mode 100644 index 0000000..1aa5689 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/softmax.h @@ -0,0 +1,366 @@ +#pragma once + +#include "../../cudnn_frontend_Heuristics.h" +#include "../../cudnn_frontend_Logging.h" + +#include "../graph_helpers.h" +#include "../node_interface.h" + +#include "pointwise.h" +#include "reduction.h" + +namespace cudnn_frontend::graph { + +// Base class for nodes that represent a softmax operation. +// This is either implemented as a composite of pointwise/reduction operations (CompositeSoftmaxNode) +// or as a single unified backend operation (UnifiedSoftmaxNode). +template <typename DerivedT> +class SoftmaxNodeBase : public NodeCRTP<DerivedT> { + protected: + using input_names = Softmax_attributes::input_names; + using output_names = Softmax_attributes::output_names; + + public: + Softmax_attributes attributes; + + SoftmaxNodeBase(Softmax_attributes&& attributes_, detail::Context const& context) + : NodeCRTP<DerivedT>(context), attributes(std::move(attributes_)) {} + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for Softmax node " << attributes.name); + + attributes.fill_from_context(this->context); + + auto input_tensor = attributes.inputs.at(Softmax_attributes::input_names::P); + auto output_tensor = attributes.outputs.at(Softmax_attributes::output_names::S); + + // Only infer dims and strides if user did not set them + if (output_tensor->get_dim().empty()) { + output_tensor->set_dim(input_tensor->get_dim()); + } + + if (output_tensor->get_stride().empty()) { + output_tensor->set_stride(input_tensor->get_stride()); + } + + return {error_code_t::OK, ""}; + } + + protected: + bool + has_sink() const { + auto sink_it = attributes.inputs.find(Softmax_attributes::input_names::SINK); + return ((sink_it) != attributes.inputs.end() && sink_it->second != nullptr); + } + + bool + has_stats() const { + auto stats_it = attributes.outputs.find(Softmax_attributes::output_names::Stats); + return ((stats_it) != attributes.outputs.end() && stats_it->second != nullptr); + } + + bool + has_max() const { + auto max_it = attributes.outputs.find(Softmax_attributes::output_names::Max); + return ((max_it) != attributes.outputs.end() && max_it->second != nullptr); + } + + bool + has_sum_exp() const { + auto sum_exp_it = attributes.outputs.find(Softmax_attributes::output_names::Sum_exp); + return ((sum_exp_it) != attributes.outputs.end() && sum_exp_it->second != nullptr); + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + void + serialize(json& j) const override final { + j = attributes; + } +#endif +}; + +// Fallback implementation of softmax node that represents the operation as a series of pointwise and reduction +// operations. This is used for cuDNN versions before v9.21.0. +// Only certain combinations of outputs are allowed. +class CompositeSoftmaxNode : public SoftmaxNodeBase<CompositeSoftmaxNode> { + public: + CompositeSoftmaxNode(Softmax_attributes&& attributes_, detail::Context const& context) + : SoftmaxNodeBase(std::move(attributes_), context) {} + + Type + getType() override final { + return Type::COMPOSITE; + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating CompositeSoftmaxNode " << attributes.name); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + !((!has_stats() && !has_max() && !has_sum_exp()) || // + (has_stats() && !has_max() && !has_sum_exp()) || // + (!has_stats() && has_max() && has_sum_exp())), + error_code_t::GRAPH_NOT_SUPPORTED, + "CompositeSoftmaxNode can only output certain combinations of stats, max and sum_exp: " + "stats only, max and sum_exp only, or none of the above."); + + return {error_code_t::OK, ""}; + } + + error_t + expand_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for CompositeSoftmaxNode " << attributes.name); + + attributes.fill_from_context(context); + + // Fill properties of virtual tensors + auto const p_dim = attributes.inputs[Softmax_attributes::input_names::P]->get_dim(); + auto b = p_dim[0]; + auto h = p_dim[1]; + auto s_q = p_dim[2]; + + std::shared_ptr<Tensor_attributes> max_output; + if (has_max()) { + max_output = attributes.outputs[Softmax_attributes::output_names::Max]; + } else { + max_output = std::make_shared<Tensor_attributes>(); + max_output->set_is_virtual(true); + } + //////////////// TODO ////////////////////////// + // Check Stride (Before setting dimension?) + if (max_output->get_dim().empty()) { + max_output->set_dim({b, h, s_q, 1}); + } + if (max_output->get_stride().empty()) { + max_output->set_stride({h * s_q, s_q, 1, 1}); + } + + auto max_attributes = Reduction_attributes().set_name("Max").set_mode(ReductionMode_t::MAX); + // If sink tensor is present, we also need to take a pointwise max with sink + if (has_sink()) { + auto s_max = reduction(attributes.inputs[Softmax_attributes::input_names::P], max_attributes); + s_max->set_name("s_max"); + + auto sink_tensor = attributes.inputs[Softmax_attributes::input_names::SINK]; + auto sink_attributes = Pointwise_attributes().set_name("max_sink").set_mode(PointwiseMode_t::MAX); + pointwise(s_max, sink_tensor, sink_attributes, max_output); + } else { + // Special non-functional-style call. Needed because output already created and provided to user. + reduction(attributes.inputs[Softmax_attributes::input_names::P], max_attributes, max_output); + } + + auto sub_attributes = Pointwise_attributes().set_name("sub").set_mode(PointwiseMode_t::SUB); + auto const& sub_output = + pointwise(attributes.inputs[Softmax_attributes::input_names::P], max_output, sub_attributes); + sub_output->set_name("sub_M"); + + auto exp_attributes = Pointwise_attributes().set_name("exp").set_mode(PointwiseMode_t::EXP); + auto const& exp_output = pointwise(sub_output, exp_attributes); + exp_output->set_name("exp_sub_M"); + + std::shared_ptr<Tensor_attributes> sum_output; + if (has_sum_exp()) { + sum_output = attributes.outputs[Softmax_attributes::output_names::Sum_exp]; + } else { + sum_output = std::make_shared<Tensor_attributes>(); + sum_output->set_is_virtual(true); + } + sum_output->set_name("SumExp"); + if (sum_output->get_dim().empty()) { + sum_output->set_dim({b, h, s_q, 1}); + } + if (sum_output->get_stride().empty()) { + sum_output->set_stride({h * s_q, s_q, 1, 1}); + } + auto sum_attributes = Reduction_attributes().set_name("sum").set_mode(ReductionMode_t::ADD); + // If sink tensor is present, also subtract it and take its exp + if (has_sink()) { + auto sink_tensor = attributes.inputs[Softmax_attributes::input_names::SINK]; + auto sub_sink = pointwise(sink_tensor, max_output, sub_attributes); + sub_sink->set_name("sub_sink"); + + auto exp_sink = pointwise(sub_sink, exp_attributes); + exp_sink->set_name("exp_sink"); + + auto temp_sum = reduction(exp_output, sum_attributes); + temp_sum->set_name("SumExp_elements").set_dim({b, h, s_q, 1}).set_stride({h * s_q, s_q, 1, 1}); + + auto add_attributes = Pointwise_attributes().set_name("add_sink").set_mode(PointwiseMode_t::ADD); + pointwise(temp_sum, exp_sink, add_attributes, sum_output); + } else { + reduction(exp_output, sum_attributes, sum_output); + } + + // Set to virtual when: + // - softmax stats in not requested + // - max and sum_exp are not requested + if (!has_stats() && !has_max() && !has_sum_exp()) { + auto softmax_stats = std::make_shared<Tensor_attributes>(); + softmax_stats->set_is_virtual(true); + attributes.outputs[Softmax_attributes::output_names::Stats] = softmax_stats; + } + + if (has_stats()) { + auto log_attributes = Pointwise_attributes().set_name("log").set_mode(PointwiseMode_t::LOG); + auto const& log_output = pointwise(sum_output, log_attributes); + log_output->set_dim({b, h, s_q, 1}).set_stride({h * s_q, s_q, 1, 1}); + + auto add_attributes = Pointwise_attributes().set_name("add").set_mode(PointwiseMode_t::ADD); + // Special non-functional-style call. Needed because output already created and provided to user. + pointwise( + max_output, log_output, add_attributes, attributes.outputs[Softmax_attributes::output_names::Stats]); + } + + auto div_attributes = Pointwise_attributes().set_name("div").set_mode(PointwiseMode_t::DIV); + // Special non-functional-style call. Needed because output already created and provided to user. + pointwise(exp_output, sum_output, div_attributes, attributes.outputs[Softmax_attributes::output_names::S]); + + return {error_code_t::OK, ""}; + } +}; + +// Newer implementation of softmax node that represents the operation as a single backend operation. +// This is used for cuDNN versions v9.21.0 and later. +class UnifiedSoftmaxNode : public SoftmaxNodeBase<UnifiedSoftmaxNode> { + public: + UnifiedSoftmaxNode(Softmax_attributes&& attributes_, detail::Context const& context) + : SoftmaxNodeBase(std::move(attributes_), context) {} + + Type + getType() override final { + return Type::SOFTMAX; + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating UnifiedSoftmaxNode " << attributes.name); + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations, + std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final { + getLogger() << "[cudnn_frontend] INFO: " << "Building UnifiedSoftmaxNode operations " << attributes.name + << std::endl; + auto cudnn_ver_error = error_t{error_code_t::GRAPH_NOT_SUPPORTED, "UnifiedSoftmaxNode requires cuDNN v9.21.0"}; + +#if (CUDNN_VERSION >= 92100) + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(92100, cudnn_ver_error); + CUDNN_FRONTEND_UNUSED(operations); + auto softmax_operation = + make_shared_backend_pointer((cudnnBackendDescriptorType_t)CUDNN_BACKEND_OPERATION_SOFTMAX_DESCRIPTOR); + + // Set input tensor P + auto P = attributes.inputs.find(Softmax_attributes::input_names::P)->second; + auto backend_p = tensors[P->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(softmax_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SOFTMAX_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_p)); + + // Set output tensor S + auto S = attributes.outputs.find(Softmax_attributes::output_names::S)->second; + auto backend_s = tensors[S->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(softmax_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SOFTMAX_YDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_s)); + + // Set optional SINK tensor if present + if (has_sink()) { + auto SINK = attributes.inputs.find(Softmax_attributes::input_names::SINK)->second; + auto backend_sink = tensors[SINK->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(softmax_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SOFTMAX_SINK_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_sink)); + } + + // Set optional Stats output tensor if present + if (has_stats()) { + auto Stats = attributes.outputs.find(Softmax_attributes::output_names::Stats)->second; + auto backend_stats = tensors[Stats->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(softmax_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SOFTMAX_STATS_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_stats)); + } + + // Set optional Max output tensor if present + if (has_max()) { + auto Max = attributes.outputs.find(Softmax_attributes::output_names::Max)->second; + auto backend_max = tensors[Max->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(softmax_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SOFTMAX_MAX_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_max)); + } + + // Set optional Sum_exp output tensor if present + if (has_sum_exp()) { + auto Sum_exp = attributes.outputs.find(Softmax_attributes::output_names::Sum_exp)->second; + auto backend_sum_exp = tensors[Sum_exp->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(softmax_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SOFTMAX_SUM_EXP_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_sum_exp)); + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(softmax_operation->get_backend_descriptor())); + + raw_operations.push_back(softmax_operation); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; +#else + CUDNN_FRONTEND_UNUSED(uids_involved_in_operations); + CUDNN_FRONTEND_UNUSED(operations); + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FRONTEND_UNUSED(tensors); + return cudnn_ver_error; +#endif // CUDNN_VERSION >= 92100 + } +}; + +// Factory function implementation for INode::softmax +// This function will create the appropriate node based on the cuDNN version. +inline void +INode::softmax(std::shared_ptr<Tensor_attributes> p, + Softmax_attributes attributes, + std::shared_ptr<Tensor_attributes> s, + std::shared_ptr<Tensor_attributes> stats, + std::shared_ptr<Tensor_attributes> max, + std::shared_ptr<Tensor_attributes> sum_exp) { + attributes.inputs[Softmax_attributes::input_names::P] = p; + attributes.outputs[Softmax_attributes::output_names::S] = s; + if (stats) { + attributes.outputs[Softmax_attributes::output_names::Stats] = stats; + } + if (max) { + attributes.outputs[Softmax_attributes::output_names::Max] = max; + } + if (sum_exp) { + attributes.outputs[Softmax_attributes::output_names::Sum_exp] = sum_exp; + } + + // Newer versions of cuDNN can represent the softmax as a single operation. + if (std::min(detail::get_compiled_version(), detail::get_backend_version()) >= 92100) { + sub_nodes.emplace_back(std::make_unique<UnifiedSoftmaxNode>(std::move(attributes), context)); + } else { + sub_nodes.emplace_back(std::make_unique<CompositeSoftmaxNode>(std::move(attributes), context)); + } +} + +} // namespace cudnn_frontend::graph diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/transpose.h b/deps/cudnn-frontend/include/cudnn_frontend/node/transpose.h new file mode 100644 index 0000000..7f764be --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/transpose.h @@ -0,0 +1,164 @@ +#pragma once + +#include "../../cudnn_frontend_Logging.h" + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend::graph { + +class TransposeNode : public NodeCRTP<TransposeNode> { + public: + Transpose_attributes attributes; + + TransposeNode(Transpose_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::TRANSPOSE; + } + + error_t + infer_properties_node() override final { + getLogger() << "[cudnn_frontend] INFO: Inferrencing properties for transpose node " << attributes.name + << std::endl; + + attributes.fill_from_context(context); + + auto const& X_tensor = attributes.inputs[Transpose_attributes::input_names::X]; + auto Y_tensor = attributes.outputs[Transpose_attributes::output_names::Y]; + + // Get input properties + auto const& input_dim = X_tensor->get_dim(); + auto const& input_stride = X_tensor->get_stride(); + auto const& input_data_type = X_tensor->get_data_type(); + auto const& permutation = attributes.get_permutation(); + + // Validate permutation + RETURN_CUDNN_FRONTEND_ERROR_IF( + permutation.empty(), error_code_t::ATTRIBUTE_NOT_SET, "Permutation must be set for transpose operation."); + + RETURN_CUDNN_FRONTEND_ERROR_IF(permutation.size() != input_dim.size(), + error_code_t::INVALID_VALUE, + "Permutation size must match input tensor dimensionality."); + + // Check that permutation is a valid permutation (contains each index 0 to n-1 exactly once) + std::vector<bool> seen(permutation.size(), false); + for (auto idx : permutation) { + RETURN_CUDNN_FRONTEND_ERROR_IF(idx < 0 || idx >= static_cast<int64_t>(permutation.size()), + error_code_t::INVALID_VALUE, + "Permutation indices must be in range [0, n-1]."); + RETURN_CUDNN_FRONTEND_ERROR_IF( + seen[idx], error_code_t::INVALID_VALUE, "Permutation indices must be unique."); + seen[idx] = true; + } + + // Infer output dimensions by permuting input dimensions + std::vector<int64_t> output_dim(input_dim.size()); + for (size_t i = 0; i < permutation.size(); ++i) { + output_dim[i] = input_dim[permutation[i]]; + } + + // Infer output strides by permuting input strides + std::vector<int64_t> output_stride(input_stride.size()); + for (size_t i = 0; i < permutation.size(); ++i) { + output_stride[i] = input_stride[permutation[i]]; + } + + // Set output tensor properties + if (Y_tensor->get_dim().empty()) { + Y_tensor->set_dim(output_dim); + } + if (Y_tensor->get_stride().empty()) { + Y_tensor->set_stride(output_stride); + } + if (Y_tensor->get_data_type() == DataType_t::NOT_SET) { + Y_tensor->set_data_type(input_data_type); + } + + RETURN_CUDNN_FRONTEND_ERROR_IF(Y_tensor->get_data_type() != input_data_type, + error_code_t::INVALID_VALUE, + "Output and input tensor data types must match for transpose operation."); + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations, + std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final { + getLogger() << "[cudnn_frontend] INFO: " << "Building TransposeNode operations " << attributes.name + << std::endl; + +#if (CUDNN_VERSION >= 92200) + // cuDNN >= 9.22.0: Use native backend transpose operation + auto cudnn_ver_error = error_t{error_code_t::GRAPH_NOT_SUPPORTED, "Transpose requires cuDNN v9.22.0"}; + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(92200, cudnn_ver_error); + CUDNN_FRONTEND_UNUSED(operations); + + auto transpose_operation = make_shared_backend_pointer(CUDNN_BACKEND_OPERATION_TRANSPOSE_DESCRIPTOR); + + // Set input tensor + auto X = attributes.inputs.at(Transpose_attributes::input_names::X); + auto backend_x = tensors[X->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(transpose_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_TRANSPOSE_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_x)); + + // Set output tensor + auto Y = attributes.outputs.at(Transpose_attributes::output_names::Y); + auto backend_y = tensors[Y->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(transpose_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_TRANSPOSE_YDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_y)); + + // Set permutation + auto permutation = attributes.get_permutation(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(transpose_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_TRANSPOSE_PERMUTATION, + CUDNN_TYPE_INT64, + permutation.size(), + permutation.data())); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(transpose_operation->get_backend_descriptor())); + + raw_operations.push_back(transpose_operation); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); +#else + CUDNN_FRONTEND_UNUSED(operations); + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FRONTEND_UNUSED(tensors); + CUDNN_FRONTEND_UNUSED(uids_involved_in_operations); + return {error_code_t::GRAPH_NOT_SUPPORTED, "Transpose operation requires cuDNN version >= 9.22.0"}; +#endif + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "TRANSPOSE"})"_json); + } +#endif +}; + +inline std::shared_ptr<Tensor_attributes> +INode::transpose(std::shared_ptr<Tensor_attributes> input, Transpose_attributes attributes) { + attributes.inputs[Transpose_attributes::input_names::X] = input; + auto Y = attributes.outputs[Transpose_attributes::output_names::Y] = output_tensor(attributes.name + "::Y"); + + sub_nodes.emplace_back(std::make_unique<TransposeNode>(std::move(attributes), context)); + return Y; +} + +} // namespace cudnn_frontend::graph |
