aboutsummaryrefslogtreecommitdiff
path: root/deps/cudnn-frontend/include/cudnn_frontend/node/block_scale_quantize.h
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-07-02 19:44:08 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-07-02 19:44:08 +0200
commit6a10d3577232f1fd646a20f22a4a1ab4ec29f28a (patch)
treee4a077b064876ddb39144935fa256a4ab0692488 /deps/cudnn-frontend/include/cudnn_frontend/node/block_scale_quantize.h
parent98a713e86c23bc6098bf7a5558507a9f27a7a1c1 (diff)
downloadffwd-6a10d3577232f1fd646a20f22a4a1ab4ec29f28a.tar.gz
ffwd-6a10d3577232f1fd646a20f22a4a1ab4ec29f28a.zip
cuda: add cuDNN fused attention path with runtime fallback
Diffstat (limited to 'deps/cudnn-frontend/include/cudnn_frontend/node/block_scale_quantize.h')
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/node/block_scale_quantize.h205
1 files changed, 205 insertions, 0 deletions
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/block_scale_quantize.h b/deps/cudnn-frontend/include/cudnn_frontend/node/block_scale_quantize.h
new file mode 100644
index 0000000..494ea14
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/node/block_scale_quantize.h
@@ -0,0 +1,205 @@
+#pragma once
+
+#include "../../cudnn_frontend_Logging.h"
+#include "../../cudnn_frontend_shim.h"
+
+#include "../graph_helpers.h"
+#include "../node_interface.h"
+
+namespace cudnn_frontend {
+
+namespace graph {
+
+class BlockScaleQuantizeNode : public NodeCRTP<BlockScaleQuantizeNode> {
+ public:
+ Block_scale_quantize_attributes attributes;
+
+ BlockScaleQuantizeNode(Block_scale_quantize_attributes&& attributes_, detail::Context const& context)
+ : NodeCRTP(context), attributes(std::move(attributes_)) {}
+
+ Type
+ getType() override final {
+ return Type::BLOCK_SCALE_QUANTIZE;
+ }
+
+ error_t
+ pre_validate_node() const override final {
+ getLogger() << "[cudnn_frontend] INFO: " << "Validating BlockScaleQuantizeNode " << attributes.name
+ << std::endl;
+
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ !attributes.block_size.has_value(), error_code_t::ATTRIBUTE_NOT_SET, "Block size not set.");
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ infer_properties_node() override final {
+ getLogger() << "[cudnn_frontend] INFO: Inferencing properties for BlockScaleQuantizeNode " << attributes.name
+ << std::endl;
+
+ attributes.fill_from_context(context);
+
+ auto X = attributes.inputs[Block_scale_quantize_attributes::input_names::X];
+ auto Y = attributes.outputs[Block_scale_quantize_attributes::output_names::Y];
+ auto scale = attributes.outputs[Block_scale_quantize_attributes::output_names::scale];
+
+ // Block scale quantize requires the block scale axis to be packed
+ auto infer_strides_transposed = [&X](std::shared_ptr<Tensor_attributes>& T,
+ std::optional<int64_t> const& axis) {
+ auto const& dim = T->get_dim();
+ auto const& X_dim = X->get_dim();
+ auto const& X_stride = X->get_stride();
+
+ std::vector<int64_t> indices(X_stride.size());
+ std::iota(indices.begin(), indices.end(), 0);
+ // Sort indices based on stride values in descending order
+ std::sort(indices.begin(), indices.end(), [&X_dim, &X_stride](int64_t i, int64_t j) {
+ // Prioritize singleton dimensions
+ if (X_stride[i] == X_stride[j]) {
+ return (X_dim[i] == 1) && (X_dim[j] != 1);
+ }
+ return X_stride[i] < X_stride[j];
+ });
+ if (axis) {
+ // Rotate left until the axis is the packed dim
+ std::rotate(indices.begin(), std::find(indices.begin(), indices.end(), axis.value()), indices.end());
+ }
+ std::vector<int64_t> stride_order(X_stride.size());
+ for (size_t i = 0; i < indices.size(); ++i) {
+ stride_order[indices[i]] = i;
+ }
+ T->set_stride(detail::generate_stride(dim, stride_order));
+ };
+
+ // Only infer dims and strides if user did not set them
+ if (Y->get_dim().empty()) {
+ Y->set_dim(X->get_dim());
+ }
+ if (Y->get_stride().empty()) {
+ if (attributes.transpose) {
+ infer_strides_transposed(Y, attributes.axis);
+ } else {
+ Y->set_stride(X->get_stride());
+ }
+ }
+
+ // Only infer dims and strides if user did not set them
+ if (scale->get_dim().empty()) {
+ auto scale_dim = X->get_dim();
+ if (attributes.axis) {
+ scale_dim[attributes.axis.value()] /= attributes.block_size.value();
+ } else {
+ scale_dim.back() /= attributes.block_size.value();
+ }
+ scale->set_dim(scale_dim);
+ }
+ if (scale->get_stride().empty()) {
+ if (attributes.transpose) {
+ infer_strides_transposed(scale, attributes.axis);
+ } else {
+ auto const& scale_dim = scale->get_dim();
+ std::vector<int64_t> stride_order;
+ CHECK_CUDNN_FRONTEND_ERROR(
+ detail::generate_stride_order_preserving_format(X->get_stride(), scale_dim.size(), stride_order));
+ scale->set_stride(detail::generate_stride(scale_dim, stride_order));
+ }
+ }
+
+ return {error_code_t::OK, ""};
+ }
+
+ error_t
+ create_cudnn_operations(
+ std::unordered_set<Tensor_attributes::uid_t>& uids_involved_in_operations,
+ std::vector<std::shared_ptr<cudnn_frontend::Operation>>& operations,
+ managed_backend_descriptor_t& raw_operations,
+ std::unordered_map<int64_t, std::shared_ptr<cudnn_frontend::Tensor>>& tensors) const override final {
+ getLogger() << "[cudnn_frontend] INFO: " << "Building BlockScaleQuantizeNode operations " << attributes.name
+ << std::endl;
+ auto cudnn_ver_error = error_t{error_code_t::GRAPH_NOT_SUPPORTED, "Block scale quantize requires cuDNN v9.7.0"};
+
+#if (CUDNN_VERSION >= 90700) // TODO: v9.99 is new feature branch; switch to release branch when ready
+ NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(90700, cudnn_ver_error);
+ CUDNN_FRONTEND_UNUSED(operations);
+ auto block_scale_quantize_operation = make_shared_backend_pointer(
+ (cudnnBackendDescriptorType_t)CUDNN_BACKEND_OPERATION_BLOCK_SCALE_QUANTIZE_DESCRIPTOR);
+
+ auto X = attributes.inputs.find(Block_scale_quantize_attributes::input_names::X)->second;
+ auto backend_x = tensors[X->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(block_scale_quantize_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_BLOCK_SCALE_QUANTIZE_XDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_x));
+
+ auto Y = attributes.outputs.find(Block_scale_quantize_attributes::output_names::Y)->second;
+ auto backend_y = tensors[Y->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(block_scale_quantize_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_BLOCK_SCALE_QUANTIZE_YDESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_y));
+
+ auto scale = attributes.outputs.find(Block_scale_quantize_attributes::output_names::scale)->second;
+ auto backend_scale = tensors[scale->get_uid()]->get_desc()->get_backend_descriptor();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(block_scale_quantize_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_BLOCK_SCALE_QUANTIZE_SCALE_DESC,
+ CUDNN_TYPE_BACKEND_DESCRIPTOR,
+ 1,
+ &backend_scale));
+
+ cudnnDataType_t cudnn_data_type;
+ _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.compute_data_type, cudnn_data_type));
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(block_scale_quantize_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_BLOCK_SCALE_QUANTIZE_MATH_PREC,
+ CUDNN_TYPE_DATA_TYPE,
+ 1,
+ &cudnn_data_type));
+
+ int32_t block_size = attributes.block_size.value();
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(block_scale_quantize_operation->get_backend_descriptor(),
+ CUDNN_ATTR_OPERATION_BLOCK_SCALE_QUANTIZE_BLOCK_SIZE,
+ CUDNN_TYPE_INT32,
+ 1,
+ &block_size));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(block_scale_quantize_operation->get_backend_descriptor()));
+
+ raw_operations.push_back(block_scale_quantize_operation);
+
+ auto const& non_virtual_uids = attributes.get_non_virtual_uids();
+ uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end());
+ return {error_code_t::OK, ""};
+#else
+ CUDNN_FRONTEND_UNUSED(uids_involved_in_operations);
+ CUDNN_FRONTEND_UNUSED(operations);
+ CUDNN_FRONTEND_UNUSED(raw_operations);
+ CUDNN_FRONTEND_UNUSED(tensors);
+ return cudnn_ver_error;
+#endif
+ }
+
+#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB
+ virtual void
+ serialize(json& j) const override final {
+ j = attributes;
+ j.update(R"( {"tag": "BLOCK_SCALE_QUANTIZE"})"_json);
+ }
+#endif
+};
+
+inline void
+INode::block_scale_quantize(std::shared_ptr<Tensor_attributes> x,
+ Block_scale_quantize_attributes attributes,
+ std::shared_ptr<Tensor_attributes> y,
+ std::shared_ptr<Tensor_attributes> scale) {
+ attributes.inputs[Block_scale_quantize_attributes::input_names::X] = x;
+ attributes.outputs[Block_scale_quantize_attributes::output_names::Y] = y;
+ attributes.outputs[Block_scale_quantize_attributes::output_names::scale] = scale;
+ sub_nodes.emplace_back(std::make_unique<BlockScaleQuantizeNode>(std::move(attributes), context));
+}
+
+} // namespace graph
+
+} // namespace cudnn_frontend