aboutsummaryrefslogtreecommitdiff
path: root/deps/cudnn-frontend/include/cudnn_frontend/node/batchnorm.h
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-07-02 19:44:08 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-07-02 19:44:08 +0200
commit6a10d3577232f1fd646a20f22a4a1ab4ec29f28a (patch)
treee4a077b064876ddb39144935fa256a4ab0692488 /deps/cudnn-frontend/include/cudnn_frontend/node/batchnorm.h
parent98a713e86c23bc6098bf7a5558507a9f27a7a1c1 (diff)
downloadffwd-6a10d3577232f1fd646a20f22a4a1ab4ec29f28a.tar.gz
ffwd-6a10d3577232f1fd646a20f22a4a1ab4ec29f28a.zip
cuda: add cuDNN fused attention path with runtime fallback
Diffstat (limited to 'deps/cudnn-frontend/include/cudnn_frontend/node/batchnorm.h')
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/node/batchnorm.h268
1 files changed, 268 insertions, 0 deletions
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/batchnorm.h b/deps/cudnn-frontend/include/cudnn_frontend/node/batchnorm.h
new file mode 100644
index 0000000..69d232b
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/node/batchnorm.h
@@ -0,0 +1,268 @@
+#pragma once
+
+#include "../graph_helpers.h"
+#include "../node_interface.h"
+
+namespace cudnn_frontend {
+
+namespace graph {
+class BatchNormNode : public NodeCRTP<BatchNormNode> {
+ public:
+ Batchnorm_attributes attributes;
+
+ BatchNormNode(Batchnorm_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP(context), attributes(std::move(attributes_)) {}
+
+ Type
+ getType() override final {
+ return Type::BATCHNORM;
+ }
+
+ error_t
+ infer_properties_node() override final {
+ CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for batchnorm node " << attributes.name);
+
+ attributes.fill_from_context(context);
+
+ auto X = attributes.inputs[Batchnorm_attributes::input_names::X];
+ auto Y = attributes.outputs[Batchnorm_attributes::output_names::Y];
+ // Only infer dims and strides if user did not set them
+ if (Y->get_dim().empty()) {
+ Y->set_dim(X->get_dim());
+ }
+ if (Y->get_stride().empty()) {
+ Y->set_stride(X->get_stride());
+ }
+
+ // Set channel length tensors
+ auto const x_tensor_dim = X->get_dim();
+ auto infer_per_channel_tensors = [&x_tensor_dim](std::shared_ptr<Tensor_attributes>& T) {
+ auto tensor_dim = T->get_dim();
+ // Only infer dims and strides if user did not set them
+ if (tensor_dim.empty()) {
+ tensor_dim.resize(x_tensor_dim.size(), 1);
+ tensor_dim[1] = x_tensor_dim[1];
+ T->set_dim(tensor_dim);
+ }
+ if (T->get_stride().empty()) {
+ auto const& T_dim = T->get_dim();
+ // Default to NHWC
+ auto const& stride_order = detail::generate_NHWC_stride_order(T_dim.size());
+ T->set_stride(detail::generate_stride(T_dim, stride_order));
+ }
+ };
+ infer_per_channel_tensors(attributes.outputs[Batchnorm_attributes::output_names::MEAN]);
+ infer_per_channel_tensors(attributes.outputs[Batchnorm_attributes::output_names::INV_VARIANCE]);
+
+ auto has_running_stats = attributes.inputs[Batchnorm_attributes::input_names::PREV_RUNNING_MEAN] ||
+ attributes.inputs[Batchnorm_attributes::input_names::PREV_RUNNING_VAR];
+
+ if (has_running_stats) {
+ infer_per_channel_tensors(attributes.outputs[Batchnorm_attributes::output_names::NEXT_RUNNING_MEAN]);
+ infer_per_channel_tensors(attributes.outputs[Batchnorm_attributes::output_names::NEXT_RUNNING_VAR]);
+ }
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ create_cudnn_operations(
+ std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations,
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations,
+ managed_backend_descriptor_t& raw_operations,
+ std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final {
+ CUDNN_FRONTEND_UNUSED(raw_operations);
+ CUDNN_FE_LOG_LABEL("INFO: Building BatchNormNode operations " << attributes.name << " ");
+
+ // Create operation by directly calling cuDNN backend API
+ Operation_v8 batchnorm_operation;
+
+ _CUDNN_CHECK_CUDNN_ERROR(
+ batchnorm_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_NORM_FORWARD_DESCRIPTOR));
+
+ // Set norm mode to BATCH_NORM
+ cudnnBackendNormMode_t cudnn_norm_mode;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormMode_t::BATCH_NORM, cudnn_norm_mode));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_MODE,
+ CUDNN_TYPE_NORM_MODE,
+ 1,
+ &cudnn_norm_mode));
+
+ // Set forward phase to TRAINING
+ cudnnBackendNormFwdPhase_t cudnn_norm_fwd_phase;
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormFwdPhase_t::TRAINING, cudnn_norm_fwd_phase));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_PHASE,
+ CUDNN_TYPE_NORM_FWD_PHASE,
+ 1,
+ &cudnn_norm_fwd_phase));
+
+ // Set input tensor X
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Batchnorm_attributes::input_names::X);
+ auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_XDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &x_desc));
+
+ // Set saved mean and inv_variance
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(MEAN, Batchnorm_attributes::output_names::MEAN);
+ auto mean_desc = tensors.at(MEAN->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_MEAN_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &mean_desc));
+
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(INV_VARIANCE, Batchnorm_attributes::output_names::INV_VARIANCE);
+ auto inv_var_desc = tensors.at(INV_VARIANCE->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_INV_VARIANCE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &inv_var_desc));
+
+ // Set scale and bias tensors
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SCALE, Batchnorm_attributes::input_names::SCALE);
+ auto scale_desc = tensors.at(SCALE->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_SCALE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &scale_desc));
+
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(BIAS, Batchnorm_attributes::input_names::BIAS);
+ auto bias_desc = tensors.at(BIAS->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_BIAS_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &bias_desc));
+
+ // Set epsilon tensor
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(EPSILON, Batchnorm_attributes::input_names::EPSILON);
+ auto epsilon_desc = tensors.at(EPSILON->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_EPSILON_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &epsilon_desc));
+
+ // Check for running stats
+ bool has_running_stats = true;
+ auto it = attributes.inputs.find(Batchnorm_attributes::input_names::PREV_RUNNING_MEAN);
+ if (it == attributes.inputs.end() || it->second == nullptr) {
+ has_running_stats = false;
+ }
+
+ if (has_running_stats) {
+ // Set momentum (exp decay factor)
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(MOMENTUM, Batchnorm_attributes::input_names::MOMENTUM);
+ auto momentum_desc = tensors.at(MOMENTUM->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_EXP_AVG_FACTOR_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &momentum_desc));
+
+ // Set prev running mean and var
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(PREV_RUNNING_MEAN,
+ Batchnorm_attributes::input_names::PREV_RUNNING_MEAN);
+ auto prev_mean_desc = tensors.at(PREV_RUNNING_MEAN->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_INPUT_RUNNING_MEAN_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &prev_mean_desc));
+
+ CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(PREV_RUNNING_VAR,
+ Batchnorm_attributes::input_names::PREV_RUNNING_VAR);
+ auto prev_var_desc = tensors.at(PREV_RUNNING_VAR->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_INPUT_RUNNING_VAR_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &prev_var_desc));
+
+ // Set next running mean and var
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(NEXT_RUNNING_MEAN,
+ Batchnorm_attributes::output_names::NEXT_RUNNING_MEAN);
+ auto next_mean_desc = tensors.at(NEXT_RUNNING_MEAN->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_OUTPUT_RUNNING_MEAN_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &next_mean_desc));
+
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(NEXT_RUNNING_VAR,
+ Batchnorm_attributes::output_names::NEXT_RUNNING_VAR);
+ auto next_var_desc = tensors.at(NEXT_RUNNING_VAR->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_OUTPUT_RUNNING_VAR_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &next_var_desc));
+ }
+
+ // Set output tensor Y
+ CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(Y, Batchnorm_attributes::output_names::Y);
+ auto y_desc = tensors.at(Y->second->get_uid())->get_raw_desc();
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_YDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &y_desc));
+
+ // Set peer stat tensors if any
+ if (!attributes.peer_stats.empty()) {
+ std::vector<cudnnBackendDescriptor_t> peer_stat_descs;
+ for (auto const& peer_stat : attributes.peer_stats) {
+ peer_stat_descs.push_back(tensors.at(peer_stat->get_uid())->get_raw_desc());
+ }
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(),
+ CUDNN_ATTR_OPERATION_NORM_FWD_PEER_STAT_DESCS,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ peer_stat_descs.size(),
+ peer_stat_descs.data()));
+ }
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(batchnorm_operation.get_raw_desc()));
+
+ operations.push_back(std::make_shared<Operation_v8>(std::move(batchnorm_operation)));
+
+ auto const& non_virtual_uids = attributes.get_non_virtual_uids();
+ uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end());
+ return {error_code_t::OK, ""};
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ j.update(R"( {"tag": "BATCHNORM"})"_json);
+ }
+#endif
+};
+
+} // namespace graph
+
+} // namespace cudnn_frontend \ No newline at end of file