From 6a10d3577232f1fd646a20f22a4a1ab4ec29f28a Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Thu, 2 Jul 2026 19:44:08 +0200 Subject: cuda: add cuDNN fused attention path with runtime fallback --- .github/workflows/release.yml | 10 + build.mk | 28 +- deps/cudnn-frontend/LICENSE.txt | 21 + deps/cudnn-frontend/README.md | 16 + deps/cudnn-frontend/include/cudnn_backend_base.h | 165 + deps/cudnn-frontend/include/cudnn_frontend.h | 152 + .../cudnn_frontend/backend/backend_descriptor.h | 138 + .../cudnn_frontend/backend/device_properties.h | 165 + .../cudnn_frontend/backend/execution_helpers.h | 125 + .../include/cudnn_frontend/backend/kernel_cache.h | 168 + .../include/cudnn_frontend/backend/plan_helpers.h | 212 + .../include/cudnn_frontend/context.h | 138 + .../include/cudnn_frontend/cudnn_interface.h | 251 + .../cudnn_frontend/experimental/attention_utils.h | 385 + .../cudnn_frontend/experimental/nvrtc_shim.h | 252 + .../experimental/oss_engine_interface.h | 261 + .../experimental/sm100_rms_norm_silu_engine.h | 697 + .../experimental/sm100_rms_norm_silu_knobs.h | 215 + .../experimental/sm100_sdpa_prefill_engine.h | 544 + .../experimental/sm90_sdpa_prefill_engine.h | 450 + .../rms_norm_silu/sm100/ln_fwd_silu_kernel.h | 468 + .../generated/rms_norm_silu/sm100/ln_headers.h | 2043 ++ .../sm100/prefill/full_seqlens/d128_fprop_kernel.h | 3479 +++ .../sm100/prefill/full_seqlens/d64_fprop_kernel.h | 3481 +++ .../sm90/prefill/full_seqlens/d128_fprop_kernel.h | 1762 ++ .../sm90/prefill/full_seqlens/d64_fprop_kernel.h | 1763 ++ .../include/cudnn_frontend/graph_helpers.h | 587 + .../include/cudnn_frontend/graph_interface.h | 3878 +++ .../include/cudnn_frontend/graph_properties.h | 3143 +++ deps/cudnn-frontend/include/cudnn_frontend/knobs.h | 254 + .../cudnn_frontend/node/adaptive_layernorm.h | 454 + .../include/cudnn_frontend/node/batchnorm.h | 268 + .../cudnn_frontend/node/batchnorm_inference.h | 156 + .../cudnn_frontend/node/block_scale_dequantize.h | 169 + .../cudnn_frontend/node/block_scale_quantize.h | 205 + .../include/cudnn_frontend/node/bn_finalize.h | 263 + .../include/cudnn_frontend/node/concatenate.h | 163 + .../include/cudnn_frontend/node/conv_dgrad.h | 205 + .../include/cudnn_frontend/node/conv_fprop.h | 254 + .../include/cudnn_frontend/node/conv_wgrad.h | 201 + .../include/cudnn_frontend/node/dbn.h | 206 + .../include/cudnn_frontend/node/dbn_weight.h | 215 + .../cudnn_frontend/node/diagonal_band_mask.h | 474 + .../include/cudnn_frontend/node/dln.h | 226 + .../include/cudnn_frontend/node/genstats.h | 147 + .../include/cudnn_frontend/node/instancenorm.h | 414 + .../include/cudnn_frontend/node/layernorm.h | 259 + .../include/cudnn_frontend/node/matmul.h | 253 + .../include/cudnn_frontend/node/matmul_fp8.h | 104 + .../cudnn_frontend/node/moe_grouped_matmul.h | 221 + .../cudnn_frontend/node/moe_grouped_matmul_bwd.h | 181 + .../include/cudnn_frontend/node/paged_cache_load.h | 153 + .../include/cudnn_frontend/node/pointwise.h | 390 + .../include/cudnn_frontend/node/reduction.h | 232 + .../include/cudnn_frontend/node/resample.h | 291 + .../include/cudnn_frontend/node/reshape.h | 156 + .../include/cudnn_frontend/node/rmsnorm.h | 406 + .../include/cudnn_frontend/node/rng.h | 187 + .../include/cudnn_frontend/node/rope.h | 146 + .../include/cudnn_frontend/node/rope_backward.h | 143 + .../node/scaled_dot_product_flash_attention.h | 2730 ++ .../include/cudnn_frontend/node/sdpa_fp8_bwd.h | 1172 + .../cudnn_frontend/node/sdpa_support_surface.h | 553 + .../include/cudnn_frontend/node/slice.h | 224 + .../include/cudnn_frontend/node/softmax.h | 366 + .../include/cudnn_frontend/node/transpose.h | 164 + .../include/cudnn_frontend/node_interface.h | 557 + deps/cudnn-frontend/include/cudnn_frontend/plans.h | 1175 + .../cudnn_frontend/thirdparty/nlohmann/LICENSE.MIT | 21 + .../cudnn_frontend/thirdparty/nlohmann/json.hpp | 26710 +++++++++++++++++++ .../cudnn_frontend/utils/attn_score_modifiers.h | 330 + .../include/cudnn_frontend/utils/serialize.h | 677 + .../include/cudnn_frontend_ConvDesc.h | 425 + .../cudnn-frontend/include/cudnn_frontend_Engine.h | 341 + .../include/cudnn_frontend_EngineConfig.h | 246 + .../include/cudnn_frontend_EngineConfigGenerator.h | 123 + .../include/cudnn_frontend_EngineFallbackList.h | 180 + .../cudnn-frontend/include/cudnn_frontend_Errata.h | 293 + .../include/cudnn_frontend_ExecutionPlan.h | 599 + .../include/cudnn_frontend_ExecutionPlanCache.h | 183 + .../include/cudnn_frontend_Filters.h | 96 + .../include/cudnn_frontend_Heuristics.h | 491 + .../include/cudnn_frontend_Logging.h | 166 + .../include/cudnn_frontend_MatMulDesc.h | 198 + .../include/cudnn_frontend_Operation.h | 3167 +++ .../include/cudnn_frontend_OperationGraph.h | 360 + .../include/cudnn_frontend_PointWiseDesc.h | 411 + .../include/cudnn_frontend_ReductionDesc.h | 245 + .../include/cudnn_frontend_Reorder_Tensor.h | 105 + .../include/cudnn_frontend_Resample.h | 494 + deps/cudnn-frontend/include/cudnn_frontend_Rng.h | 339 + .../cudnn-frontend/include/cudnn_frontend_Tensor.h | 644 + .../include/cudnn_frontend_VariantPack.h | 202 + .../include/cudnn_frontend_find_plan.h | 163 + .../include/cudnn_frontend_get_plan.h | 62 + deps/cudnn-frontend/include/cudnn_frontend_shim.h | 768 + deps/cudnn-frontend/include/cudnn_frontend_utils.h | 2722 ++ .../include/cudnn_frontend_version.h | 29 + libffwd/Makefile | 2 +- libffwd/cuda.cu | 195 +- libffwd/cuda_sdpa.cu | 343 + libffwd/cuda_sdpa.h | 69 + 102 files changed, 80725 insertions(+), 8 deletions(-) create mode 100644 deps/cudnn-frontend/LICENSE.txt create mode 100644 deps/cudnn-frontend/README.md create mode 100644 deps/cudnn-frontend/include/cudnn_backend_base.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/backend/backend_descriptor.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/backend/device_properties.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/backend/execution_helpers.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/backend/kernel_cache.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/backend/plan_helpers.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/context.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/cudnn_interface.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/experimental/attention_utils.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/experimental/nvrtc_shim.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/experimental/oss_engine_interface.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/experimental/sm100_rms_norm_silu_engine.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/experimental/sm100_rms_norm_silu_knobs.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/experimental/sm100_sdpa_prefill_engine.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/experimental/sm90_sdpa_prefill_engine.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/generated/rms_norm_silu/sm100/ln_fwd_silu_kernel.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/generated/rms_norm_silu/sm100/ln_headers.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/generated/sdpa/sm100/prefill/full_seqlens/d128_fprop_kernel.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/generated/sdpa/sm100/prefill/full_seqlens/d64_fprop_kernel.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/generated/sdpa/sm90/prefill/full_seqlens/d128_fprop_kernel.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/generated/sdpa/sm90/prefill/full_seqlens/d64_fprop_kernel.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/graph_helpers.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/graph_interface.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/graph_properties.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/knobs.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/node/adaptive_layernorm.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/node/batchnorm.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/node/batchnorm_inference.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/node/block_scale_dequantize.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/node/block_scale_quantize.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/node/bn_finalize.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/node/concatenate.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/node/conv_dgrad.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/node/conv_fprop.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/node/conv_wgrad.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/node/dbn.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/node/dbn_weight.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/node/diagonal_band_mask.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/node/dln.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/node/genstats.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/node/instancenorm.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/node/layernorm.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/node/matmul.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/node/matmul_fp8.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/node/moe_grouped_matmul.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/node/moe_grouped_matmul_bwd.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/node/paged_cache_load.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/node/pointwise.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/node/reduction.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/node/resample.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/node/reshape.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/node/rmsnorm.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/node/rng.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/node/rope.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/node/rope_backward.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/node/scaled_dot_product_flash_attention.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/node/sdpa_fp8_bwd.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/node/sdpa_support_surface.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/node/slice.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/node/softmax.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/node/transpose.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/node_interface.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/plans.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/thirdparty/nlohmann/LICENSE.MIT create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/thirdparty/nlohmann/json.hpp create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/utils/attn_score_modifiers.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend/utils/serialize.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend_ConvDesc.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend_Engine.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend_EngineConfig.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend_EngineConfigGenerator.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend_EngineFallbackList.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend_Errata.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend_ExecutionPlan.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend_ExecutionPlanCache.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend_Filters.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend_Heuristics.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend_Logging.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend_MatMulDesc.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend_Operation.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend_OperationGraph.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend_PointWiseDesc.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend_ReductionDesc.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend_Reorder_Tensor.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend_Resample.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend_Rng.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend_Tensor.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend_VariantPack.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend_find_plan.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend_get_plan.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend_shim.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend_utils.h create mode 100644 deps/cudnn-frontend/include/cudnn_frontend_version.h create mode 100644 libffwd/cuda_sdpa.cu create mode 100644 libffwd/cuda_sdpa.h diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e2ee819..1d23e1d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -144,6 +144,16 @@ jobs: with: cuda: '12.8.0' + # cuDNN headers for the fused-attention path (libffwd/cuda_sdpa.cu). Only + # the headers are needed: libcudnn is dlopen'd at runtime. The pip wheel is + # the most reliable cross-runner source for them. + - name: Install cuDNN headers + run: | + python3 -m pip install --user nvidia-cudnn-cu12 + CUDNN_INC="$(python3 -c 'import os, nvidia.cudnn; print(os.path.join(nvidia.cudnn.__path__[0], "include"))')" + test -f "$CUDNN_INC/cudnn.h" + echo "CUDNN_INC=$CUDNN_INC" >> "$GITHUB_ENV" + - name: Build release artifacts env: CC: ccache gcc diff --git a/build.mk b/build.mk index a49cebb..73eefc4 100644 --- a/build.mk +++ b/build.mk @@ -120,9 +120,33 @@ MLX_BACKEND_LDFLAGS = -framework Metal -framework Foundation \ -L$(MLXC_PREFIX)/lib -lmlxc \ -L$(MLX_PREFIX)/lib -lmlx \ -Wl,-rpath,$(MLX_PREFIX)/lib -Wl,-rpath,$(MLXC_PREFIX)/lib +# cuDNN fused attention (libffwd/cuda_sdpa.cu): compiles against the vendored +# cudnn-frontend headers and dlopens libcudnn at runtime, so only cudnn.h is +# needed and only at build time. CUDNN_INC overrides the probe below (point it +# at the directory containing cudnn.h, e.g. a pip nvidia-cudnn-cu12 wheel's +# include/); when no cudnn.h is found the CUDA backend builds without the +# fused path and keeps the built-in kernels. +CUDNN_INC ?= +ifeq ($(strip $(CUDNN_INC)),) + CUDNN_H := $(firstword $(wildcard /usr/include/cudnn.h \ + /usr/include/x86_64-linux-gnu/cudnn.h $(CUDA_HOME)/include/cudnn.h)) + ifneq ($(strip $(CUDNN_H)),) + CUDNN_INC := $(patsubst %/,%,$(dir $(CUDNN_H))) + endif +endif +ifneq ($(strip $(CUDNN_INC)),) + CUDNN_SDPA_CFLAGS = -I$(ROOT)/deps/cudnn-frontend/include -I$(CUDNN_INC) \ + -DFFWD_CUDA_SDPA=1 -DNV_CUDNN_FRONTEND_USE_DYNAMIC_LOADING +else + CUDNN_SDPA_CFLAGS = +endif + CUDA_BACKEND_CFLAGS = -DUSE_BLAS -DUSE_OPENBLAS -DUSE_GPU $(OPENBLAS_CFLAGS) \ - -I$(CUDA_HOME)/include -CUDA_BACKEND_LDFLAGS = -L$(CUDA_HOME)/lib64 $(OPENBLAS_LDFLAGS) -lcudart -lcublas -lcublasLt + -I$(CUDA_HOME)/include $(CUDNN_SDPA_CFLAGS) +# -lstdc++: cuda_sdpa.o uses the C++ runtime (cudnn-frontend), and the tools +# link with the C driver. +CUDA_BACKEND_LDFLAGS = -L$(CUDA_HOME)/lib64 $(OPENBLAS_LDFLAGS) -lcudart -lcublas -lcublasLt \ + -ldl -lstdc++ ifeq ($(BACKEND),blas) BACKEND_CFLAGS = $(BLAS_BACKEND_CFLAGS) diff --git a/deps/cudnn-frontend/LICENSE.txt b/deps/cudnn-frontend/LICENSE.txt new file mode 100644 index 0000000..eef9c44 --- /dev/null +++ b/deps/cudnn-frontend/LICENSE.txt @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ diff --git a/deps/cudnn-frontend/README.md b/deps/cudnn-frontend/README.md new file mode 100644 index 0000000..728e9f5 --- /dev/null +++ b/deps/cudnn-frontend/README.md @@ -0,0 +1,16 @@ +# cudnn-frontend (vendored) + +Header-only C++ frontend for the cuDNN graph API, used by the CUDA backend for +fused scaled-dot-product attention (`libffwd/cuda_sdpa.cu`). + +- Upstream: https://github.com/NVIDIA/cudnn-frontend +- Version: 1.25.0 (commit `91bebc8aef59722c21b39391ea5c16ed3583d959`) +- License: MIT (see `LICENSE.txt`) +- Only `include/` is vendored; samples, docs, and python bindings are dropped. + +The frontend is compiled with `NV_CUDNN_FRONTEND_USE_DYNAMIC_LOADING`, so +libcudnn is dlopen'd at runtime rather than linked. Building still requires the +cuDNN headers (`cudnn.h`); see the `CUDNN_INC` variable in `build.mk`. + +To update: replace `include/` and `LICENSE.txt` from a release tag, then record +the new version and commit here. diff --git a/deps/cudnn-frontend/include/cudnn_backend_base.h b/deps/cudnn-frontend/include/cudnn_backend_base.h new file mode 100644 index 0000000..bae2de8 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_backend_base.h @@ -0,0 +1,165 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#pragma once + +#include + +namespace cudnn_frontend { + +/// +/// OpaqueBackendPointer class +/// Holds the raws pointer to backend_descriptor +/// Usage is to wrap this into a smart pointer as +/// it helps to create and destroy the backendpointer + +class OpaqueBackendPointer { + cudnnBackendDescriptor_t m_desc = nullptr; //!< Raw void pointer + cudnnStatus_t status = CUDNN_STATUS_SUCCESS; //!< status of creation of the Descriptor + + public: + OpaqueBackendPointer(const OpaqueBackendPointer&) = delete; //!< Delete the copy constructor to prevent bad copies + OpaqueBackendPointer& + operator=(const OpaqueBackendPointer&) = delete; + OpaqueBackendPointer(OpaqueBackendPointer&&) = default; + + /** + * OpaqueBackendPointer constructor. + * Calls the cudnnBackendCreateDescriptor. Allocates memory according to the type. + */ + OpaqueBackendPointer(cudnnBackendDescriptorType_t type) { status = detail::create_descriptor(type, &m_desc); } + /** + * OpaqueBackendPointer destructor. + * Calls the cudnnBackendDestroyDescriptor. Frees memory allocated in the constructor. + */ + ~OpaqueBackendPointer() { detail::destroy_descriptor(m_desc); }; + /** + * Accessor. + * Returns the const reference to raw underlying descriptor. + * Treat it like the data() function of a smart pointer. Can be freed behind the back. + */ + cudnnBackendDescriptor_t const& + get_backend_descriptor() const { + return m_desc; + } + /** + * Accessor. + * Queries the status of the descriptor after calling the cudnnCreate. + */ + cudnnStatus_t + get_status() const { + return status; + } + /** + * Accessor. + * Queries the status of the descriptor returns true if all good. + */ + bool + is_good() const { + return status == CUDNN_STATUS_SUCCESS; + } +}; + +/*! \var A shared_ptr wrapper on top of the OpaqueBackendPointer */ +using ManagedOpaqueDescriptor = std::shared_ptr; + +/*! \fn A wrapper on top of the std::make_shared for the OpaqueBackendPointer */ +static ManagedOpaqueDescriptor +make_shared_backend_pointer(cudnnBackendDescriptorType_t type) { + return std::make_shared(type); +} + +/// +/// BackendDescriptor class +/// Holds a Managed pointer to OpaqueBackendPointer class +/// Contains the status and error message if set after any operation. +/// If exception is disabled the user must query the status after +/// build operation in order to check if the cudnn construct was built +/// correctly. +class BackendDescriptor { + public: + //! Return a string describing the backend Descriptor + virtual std::string + describe() const = 0; + + //! Get a copy of the raw descriptor pointer. Ownership is reatined and + //! gets deleted when out of scope + cudnnBackendDescriptor_t + get_raw_desc() const { + return pointer->get_backend_descriptor(); + } + + //! Current status of the descriptor + cudnnStatus_t + get_status() const { + return status; + } + + //! Set status of the descriptor + void + set_status(cudnnStatus_t const status_) const { + status = status_; + } + + //! Set Diagonistic error message. + void + set_error(const char* message) const { + err_msg = message; + } + + //! Diagonistic error message if any + const char* + get_error() const { + return err_msg.c_str(); + } + + //! Returns a copy of underlying managed descriptor + ManagedOpaqueDescriptor + get_desc() const { + return pointer; + } + + //! Initializes the underlying managed descriptor + cudnnStatus_t + initialize_managed_backend_pointer(cudnnBackendDescriptorType_t type) { + pointer = make_shared_backend_pointer(type); + return pointer->get_status(); + } + + protected: + /** + * BackendDescriptor constructor. + * Initializes the member variables as passed. + */ + BackendDescriptor(ManagedOpaqueDescriptor pointer_, cudnnStatus_t status_, std::string err_msg_) + : pointer(pointer_), status(status_), err_msg(err_msg_) {} + BackendDescriptor() = default; + + virtual ~BackendDescriptor() {}; + + ManagedOpaqueDescriptor pointer; //! Shared pointer of the OpaqueBackendPointer + + mutable cudnnStatus_t status = CUDNN_STATUS_SUCCESS; //!< Error code if any being set + mutable std::string err_msg; //!< Error message if any being set +}; + +} // namespace cudnn_frontend diff --git a/deps/cudnn-frontend/include/cudnn_frontend.h b/deps/cudnn-frontend/include/cudnn_frontend.h new file mode 100644 index 0000000..fe1a350 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend.h @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#pragma once + +// Suppress MSVC warning C4756 (overflow in constant arithmetic) that occurs +// in MSVC's header with certain compiler versions +#ifdef _MSC_VER +#pragma warning(disable : 4756) +#endif + +/*! \mainpage CUDNN FRONTEND API + * + * \section Introduction + * + * The cuDNN Frontend API is a C++ header-only library that demonstrates how to use the cuDNN C backend API. The cuDNN C + * backend API is documented in the cuDNN developer guide. + * + * \section Why use Frontend API + * + * Consider the following code snippet which showcases cudnnBackendTensor creation using the backend API and its + * equivalent front-end API code. Many among the backend constructs follow similar pattern. + * + * ~~~~~~~~~~~~~~~{.cpp} + * + * =========================================================================================== + * auto check_status = [](cudnnStatus_t status) { assert (status == CUDNN_STATUS_SUCCESS); }; + * =========================================================================================== + * // Backend code for Tensor Creation. + * cudnnBackendDescriptor_t tensor; + * + * check_status (cudnnBackendCreateDescriptor(CUDNN_BACKEND_TENSOR_DESCRIPTOR, &tensor)); + * + * check_status (cudnnBackendSetAttribute(tensor, + * CUDNN_ATTR_TENSOR_DATA_TYPE, + * CUDNN_TYPE_DATA_TYPE, + * 1, + * &data_type)); + * check_status (cudnnBackendSetAttribute(tensor, + * CUDNN_ATTR_TENSOR_DIMENSIONS, + * CUDNN_TYPE_INT64, + * tensor_dim.size(), + * tensor_dim.data())); + * check_status (cudnnBackendSetAttribute(tensor, + * CUDNN_ATTR_TENSOR_STRIDES, + * CUDNN_TYPE_INT64, + * tensor_str.size(), + * tensor_str.data())); + * check_status (cudnnBackendSetAttribute(tensor, + * CUDNN_ATTR_TENSOR_UNIQUE_ID, + * CUDNN_TYPE_INT64, + * 1, + * &id)); + * check_status (cudnnBackendSetAttribute(tensor, + * CUDNN_ATTR_TENSOR_BYTE_ALIGNMENT, + * CUDNN_TYPE_INT64, + * 1, + * &alignment)); + * check_status (cudnnBackendFinalize(tensor)); + * + * check_status (cudnnBackendDestroyDescriptor(tensor)); + * =========================================================================================== + * // FrontEnd equivalent code. + * auto tensor = cudnn_frontend::TensorBuilder() + * .setDim(tensor_dim.size(), tensor_dim.data()) + * .setStrides(tensor_str.size(), tensor_str.data()) + * .setId(id) + * .setAlignment(alignment) + * .setDataType(data_type) + * .build(); + * check_status(tensor.get_status()); + * =========================================================================================== + * + * ~~~~~~~~~~~~~~~ + * + * Frontend API serves two major purpose as a companion to the backend API. + * - Functional additions: + * - Support for auto-tuning. (cudnnGet and cudnnFind) + * - Errata filters. + * - Programmatic ease: + * - Easy memory management for the cudnnBackendDescriptor_t (RAII based classes). + * - Error handling with optional exception support. Better error messages. + * - Fewer lines of code (5-10x reduction in LOC). + * - Simpler samples on how to use the new API. + */ + +#include + +#include "cudnn_frontend_ConvDesc.h" +#include "cudnn_frontend_Heuristics.h" +#include "cudnn_frontend_Engine.h" +#include "cudnn_frontend_EngineConfig.h" +#include "cudnn_frontend_EngineFallbackList.h" +#include "cudnn_frontend_Errata.h" +#include "cudnn_frontend_ExecutionPlan.h" +#include "cudnn_frontend_Filters.h" +#include "cudnn_frontend_Operation.h" +#include "cudnn_frontend_OperationGraph.h" +#include "cudnn_frontend_Tensor.h" +#include "cudnn_frontend_VariantPack.h" +#include "cudnn_frontend_PointWiseDesc.h" +#include "cudnn_frontend_MatMulDesc.h" +#include "cudnn_frontend_Logging.h" +#include "cudnn_frontend_Reorder_Tensor.h" +#include "cudnn_frontend_ExecutionPlanCache.h" +#include "cudnn_frontend_utils.h" + +#include "cudnn_frontend_Resample.h" + +#include "cudnn_frontend/graph_interface.h" +#include "cudnn_frontend/utils/serialize.h" +#include "cudnn_frontend/backend/kernel_cache.h" +#include "cudnn_frontend/utils/attn_score_modifiers.h" +#include "cudnn_frontend/backend/device_properties.h" + +#include "cudnn_frontend_version.h" + +namespace cudnn_frontend { +using ConvDesc = ConvDesc_v8; +using ConvDescBuilder = ConvDescBuilder_v8; +using ReductionDesc = ReductionDesc_v8; +using ReductionDescBuilder = ReductionDescBuilder_v8; +using EngineHeuristicsBuilder = EngineHeuristicsBuilder_v8; +using EngineHeuristics = EngineHeuristics_v8; +using EngineBuilder = EngineBuilder_v8; +using Engine = Engine_v8; +using EngineConfig = EngineConfig_v8; +using EngineConfigBuilder = EngineConfigBuilder_v8; +using EngineFallbackList = EngineFallbackList_v8; +using EngineFallbackListBuilder = EngineFallbackListBuilder_v8; +using ResampleDesc = ResampleDesc_v8; +using ResampleDescBuilder = ResampleDescBuilder_v8; +} // namespace cudnn_frontend diff --git a/deps/cudnn-frontend/include/cudnn_frontend/backend/backend_descriptor.h b/deps/cudnn-frontend/include/cudnn_frontend/backend/backend_descriptor.h new file mode 100644 index 0000000..2cd68c0 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/backend/backend_descriptor.h @@ -0,0 +1,138 @@ +#pragma once + +#include + +#include "../graph_helpers.h" +#include "cudnn.h" + +namespace cudnn_frontend::detail { + +/** + * @brief RAII wrapper around a `cudnnBackendDescriptor_t` object. + * + * This class provides a convenient way to manage the lifetime of a `cudnnBackendDescriptor_t` + * object using the RAII (Resource Acquisition Is Initialization) idiom. It automatically + * creates the descriptor when the object is constructed and destroys it when the object + * is destroyed, ensuring proper resource management and preventing memory leaks. + * + * @note The constructor of this class does not throw exceptions. Instead, it stores the + * status of the descriptor creation operation in the `status` member variable. Callers + * should check this status and handle any errors accordingly. + */ +class backend_descriptor { + public: + /** + * @brief Constructs a `backend_descriptor` object. + * + * @param type The type of the backend descriptor to create. + */ + backend_descriptor(cudnnBackendDescriptorType_t type) { status = detail::create_descriptor(type, &desc); } + + /** + * @brief Move constructor. + * + * Transfers the ownership of the `cudnnBackendDescriptor_t` object to the new + * `backend_descriptor` instance. + * + * @param other The source `backend_descriptor` object. + */ + backend_descriptor(backend_descriptor&& other) noexcept : desc(other.desc), status(other.status) { + other.desc = nullptr; + other.status = CUDNN_STATUS_NOT_INITIALIZED; + } + + /** + * @brief Move assignment operator. + * + * Transfers the ownership of the `cudnnBackendDescriptor_t` object to the new + * `backend_descriptor` instance. + * + * @param other The source `backend_descriptor` object. + * @return A reference to the current `backend_descriptor` object. + */ + backend_descriptor& + operator=(backend_descriptor&& other) noexcept { + if (this != &other) { + desc = other.desc; + status = other.status; + + other.desc = nullptr; + } + return *this; + } + + /** + * @brief Destructor. + * + * Destroys the `cudnnBackendDescriptor_t` object and frees the associated resources. + */ + ~backend_descriptor() { + if (desc) { + detail::destroy_descriptor(desc); + } + } + + /** + * @brief Deleted copy constructor and assignment operator. + * + * `backend_descriptor` objects are not copyable to prevent unintended resource + * sharing and potential memory leaks. + */ + backend_descriptor(backend_descriptor const&) = delete; + backend_descriptor& + operator=(backend_descriptor const&) = delete; + + /** + * @brief Initializes a `backend_descriptor` object. + * + * @param type The type of the backend descriptor to create. + */ + error_t + initialize(cudnnBackendDescriptorType_t type) { + _CUDNN_CHECK_CUDNN_ERROR(detail::create_descriptor(type, &desc)); + return {error_code_t::OK, ""}; + } + + /** + * @brief Finalizes a `backend_descriptor` object. + * + */ + error_t + finalize() { + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(desc)); + return {error_code_t::OK, ""}; + } + + /** + * @brief Accessor for the underlying `cudnnBackendDescriptor_t` object. + * + * @return A const reference to `cudnnBackendDescriptor_t`, the raw pointer to the backend descriptor. + */ + cudnnBackendDescriptor_t const& + get_ptr() const { + return desc; + } + + /** + * @brief Accessor for the status of the backend descriptor creation. + * + * @return `cudnnStatus_t` The status of the backend descriptor creation operation. + */ + cudnnStatus_t + get_status() const { + return status; + } + + /** + * @brief Constructs a default `backend_descriptor` object, but without initializing descriptor + * + * Used to return an error code to user for incorrect cuDNN version + */ + backend_descriptor() = default; + + private: + cudnnBackendDescriptor_t desc = nullptr; //!< Raw pointer to the backend descriptor. + cudnnStatus_t status = CUDNN_STATUS_NOT_INITIALIZED; //!< Status of the descriptor creation operation. +}; + +} // namespace cudnn_frontend::detail \ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/backend/device_properties.h b/deps/cudnn-frontend/include/cudnn_frontend/backend/device_properties.h new file mode 100644 index 0000000..d31c555 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/backend/device_properties.h @@ -0,0 +1,165 @@ +/* + * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "../graph_helpers.h" +#include "backend_descriptor.h" + +namespace cudnn_frontend { +/// +/// DeviceProperties Class +/// Wraps the device_properties backend descriptor +/// Wraps backend utility functions for user's convenience +/// Backend accessor functions: size() +/// Contains internal utilities for device properties finalization and operation graph attributes +/// +class DeviceProperties : public detail::backend_descriptor { + public: + // Uses the default backend constructor so that we can check for initialization error during build() + DeviceProperties() = default; + + std::string + describe() const { + std::stringstream ss; + ss << "CUDNN_BACKEND_DEVICEPROP_DESCRIPTOR : " << std::endl; + return ss.str(); + } + + inline DeviceProperties& + set_device_id(int32_t device_id) { + this->device_id = device_id; + return *this; + } + + inline DeviceProperties& + set_handle(cudnnHandle_t handle) { + this->handle = handle; + return *this; + } + + // Used to check device properties status (particularly after initialization) + error_t + status() const { + if (get_status() != CUDNN_STATUS_SUCCESS) { + return {error_code_t::CUDNN_BACKEND_API_FAILED, + "CUDNN_BACKEND_DEVICEPROP_DESCRIPTOR: Check CUDNN_VERSION >= 9.8"}; + } + return {}; + } + + error_t + serialize(std::vector& serialization_buf) const { +#if (CUDNN_VERSION >= 90800) + RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 90800, + error_code_t::CUDNN_BACKEND_API_FAILED, + "CUDNN_ATTR_DEVICEPROP_JSON_REPRESENTATION is only available starting 9.8."); + + int64_t serializationSize; + _CUDNN_CHECK_CUDNN_ERROR(detail::get_attribute( + get_ptr(), CUDNN_ATTR_DEVICEPROP_JSON_REPRESENTATION, CUDNN_TYPE_CHAR, 0, &serializationSize, nullptr)); + serialization_buf.resize(static_cast(serializationSize)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::get_attribute(get_ptr(), + CUDNN_ATTR_DEVICEPROP_JSON_REPRESENTATION, + CUDNN_TYPE_CHAR, + serializationSize, + &serializationSize, + serialization_buf.data())); + return {}; +#else + (void)serialization_buf; + return {error_code_t::CUDNN_BACKEND_API_FAILED, + "CUDNN_ATTR_DEVICEPROP_JSON_REPRESENTATION is only available starting 9.8."}; +#endif + } + + error_t + deserialize(const std::vector& serialized_buf) { +#if (CUDNN_VERSION >= 90800) + RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 90800, + error_code_t::CUDNN_BACKEND_API_FAILED, + "CUDNN_ATTR_DEVICEPROP_JSON_REPRESENTATION is only available starting 9.8."); + + // Check if the device properties is already initialized + RETURN_CUDNN_FRONTEND_ERROR_IF( + get_ptr() != nullptr, error_code_t::CUDNN_BACKEND_API_FAILED, "Device properties is already initialized."); + + // Initialize the device properties descriptor + CHECK_CUDNN_FRONTEND_ERROR(initialize(CUDNN_BACKEND_DEVICEPROP_DESCRIPTOR)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(get_ptr(), + CUDNN_ATTR_DEVICEPROP_JSON_REPRESENTATION, + CUDNN_TYPE_CHAR, + serialized_buf.size(), + serialized_buf.data())); + + CHECK_CUDNN_FRONTEND_ERROR(finalize()); + return {}; +#else + (void)serialized_buf; + return {error_code_t::CUDNN_BACKEND_API_FAILED, + "CUDNN_ATTR_DEVICEPROP_JSON_REPRESENTATION is only available starting 9.8."}; +#endif + } + + // Check for both compile-time and runtime cuDNN version + error_t + build() { +#if (CUDNN_VERSION >= 90800) + RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 90800, + error_code_t::GRAPH_NOT_SUPPORTED, + "CUDNN_BACKEND_DEVICEPROP_DESCRIPTOR is only available starting 9.8."); + if (get_ptr() == nullptr) { + CHECK_CUDNN_FRONTEND_ERROR(initialize(CUDNN_BACKEND_DEVICEPROP_DESCRIPTOR)); + } + + if (handle != nullptr) { + _CUDNN_CHECK_CUDNN_ERROR( + detail::set_attribute(get_ptr(), CUDNN_ATTR_DEVICEPROP_HANDLE, CUDNN_TYPE_HANDLE, 1, &handle)); + } + + if (device_id >= 0) { + _CUDNN_CHECK_CUDNN_ERROR( + detail::set_attribute(get_ptr(), CUDNN_ATTR_DEVICEPROP_DEVICE_ID, CUDNN_TYPE_INT32, 1, &device_id)); + } + + CHECK_CUDNN_FRONTEND_ERROR(finalize()); + return {}; +#else + return {error_code_t::CUDNN_BACKEND_API_FAILED, + "CUDNN_BACKEND_DEVICEPROP_DESCRIPTOR is only available starting 9.8."}; +#endif + } + + private: + cudnnHandle_t handle = nullptr; + int32_t device_id = 0; +}; +} // namespace cudnn_frontend \ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/backend/execution_helpers.h b/deps/cudnn-frontend/include/cudnn_frontend/backend/execution_helpers.h new file mode 100644 index 0000000..7138d48 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/backend/execution_helpers.h @@ -0,0 +1,125 @@ +#pragma once + +#include + +#include "cudnn.h" + +#include "backend_descriptor.h" + +namespace cudnn_frontend::detail { +/** + * @brief Creates a CUDNN backend variant pack descriptor. + * + * This function creates a `backend_descriptor` object representing a CUDNN backend variant pack + * descriptor. The variant pack descriptor is configured with the provided device pointers, unique + * IDs, and a workspace pointer. + * + * @param[out] variant_pack The created `backend_descriptor` object representing the variant pack. + * @param device_ptrs A pointer to an array of device pointers to be associated with the variant pack. + * @param uids A vector of unique IDs to be associated with the variant pack. + * @param workspace_ptr A pointer to the workspace memory to be associated with the variant pack. + * @return `error_t` A tuple containing the error code and an optional error message. + * The error code is `error_code_t::OK` on success, or an appropriate error code on failure. + */ +inline error_t +create_variant_pack(backend_descriptor& variant_pack, + void* const* device_ptrs, + std::vector const& uids, + void* workspace_ptr) { + RETURN_CUDNN_FRONTEND_ERROR_IF(device_ptrs == nullptr && !uids.empty(), + error_code_t::INVALID_VARIANT_PACK, + "device_ptrs must be non-null when uids are provided."); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute( + variant_pack.get_ptr(), CUDNN_ATTR_VARIANT_PACK_WORKSPACE, CUDNN_TYPE_VOID_PTR, 1, &workspace_ptr)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute( + variant_pack.get_ptr(), CUDNN_ATTR_VARIANT_PACK_DATA_POINTERS, CUDNN_TYPE_VOID_PTR, uids.size(), device_ptrs)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute( + variant_pack.get_ptr(), CUDNN_ATTR_VARIANT_PACK_UNIQUE_IDS, CUDNN_TYPE_INT64, uids.size(), uids.data())); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(variant_pack.get_ptr())); + + return {error_code_t::OK, ""}; +} + +inline error_t +create_variant_pack(backend_descriptor& variant_pack, + std::vector& device_ptrs, + std::vector const& uids, + void* workspace_ptr) { + RETURN_CUDNN_FRONTEND_ERROR_IF(device_ptrs.size() != uids.size(), + error_code_t::INVALID_VARIANT_PACK, + "device_ptrs and uids must have the same length."); + return create_variant_pack(variant_pack, device_ptrs.data(), uids, workspace_ptr); +} + +inline error_t +create_variant_pack(backend_descriptor& variant_pack, + void* const* device_ptrs, + std::vector const& uids, + void* workspace_ptr, + std::vector const& override_uids, + std::vector> const& override_shapes, + std::vector> const& override_strides) { + RETURN_CUDNN_FRONTEND_ERROR_IF(device_ptrs == nullptr && !uids.empty(), + error_code_t::INVALID_VARIANT_PACK, + "device_ptrs must be non-null when uids are provided."); + auto cudnn_ver_error = error_t{error_code_t::GRAPH_NOT_SUPPORTED, "Dynamic shapes requires cuDNN v9.18.0"}; + + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(91800, cudnn_ver_error); + + CUDNN_FRONTEND_UNUSED(override_uids); + CUDNN_FRONTEND_UNUSED(override_shapes); + CUDNN_FRONTEND_UNUSED(override_strides); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute( + variant_pack.get_ptr(), CUDNN_ATTR_VARIANT_PACK_WORKSPACE, CUDNN_TYPE_VOID_PTR, 1, &workspace_ptr)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute( + variant_pack.get_ptr(), CUDNN_ATTR_VARIANT_PACK_DATA_POINTERS, CUDNN_TYPE_VOID_PTR, uids.size(), device_ptrs)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute( + variant_pack.get_ptr(), CUDNN_ATTR_VARIANT_PACK_UNIQUE_IDS, CUDNN_TYPE_INT64, uids.size(), uids.data())); + +#if (CUDNN_VERSION >= 91800) + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(variant_pack.get_ptr(), + CUDNN_ATTR_VARIANT_PACK_OVERRIDE_UNIQUE_IDS, + CUDNN_TYPE_INT64, + override_uids.size(), + override_uids.data())); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(variant_pack.get_ptr(), + CUDNN_ATTR_VARIANT_PACK_OVERRIDE_SHAPES, + CUDNN_TYPE_VOID_PTR, + 1, + (void*)&override_shapes)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(variant_pack.get_ptr(), + CUDNN_ATTR_VARIANT_PACK_OVERRIDE_STRIDES, + CUDNN_TYPE_VOID_PTR, + 1, + (void*)&override_strides)); +#endif + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(variant_pack.get_ptr())); + + return {error_code_t::OK, ""}; +} + +inline error_t +create_variant_pack(backend_descriptor& variant_pack, + std::vector& device_ptrs, + std::vector const& uids, + void* workspace_ptr, + std::vector const& override_uids, + std::vector> const& override_shapes, + std::vector> const& override_strides) { + RETURN_CUDNN_FRONTEND_ERROR_IF(device_ptrs.size() != uids.size(), + error_code_t::INVALID_VARIANT_PACK, + "device_ptrs and uids must have the same length."); + return create_variant_pack( + variant_pack, device_ptrs.data(), uids, workspace_ptr, override_uids, override_shapes, override_strides); +} + +} // namespace cudnn_frontend::detail diff --git a/deps/cudnn-frontend/include/cudnn_frontend/backend/kernel_cache.h b/deps/cudnn-frontend/include/cudnn_frontend/backend/kernel_cache.h new file mode 100644 index 0000000..ef173b2 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/backend/kernel_cache.h @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "../graph_helpers.h" +#include "backend_descriptor.h" + +namespace cudnn_frontend { +namespace graph { +class Graph; +} // namespace graph +/// +/// KernelCache Class +/// Wraps the kernel_cache backend descriptor +/// Wraps backend utility functions for user's convenience +/// Backend accessor functions: size() +/// Contains internal utilities for kernel cache finalization and operation graph attributes +/// +class KernelCache : public detail::backend_descriptor { + public: + friend class graph::Graph; + // Uses the default backend constructor so that we can check for initialization error during build() + KernelCache() : backend_descriptor() {} + + std::string + describe() const { + std::stringstream ss; + ss << "CUDNN_BACKEND_KERNEL_CACHE_DESCRIPTOR : " << std::endl; + return ss.str(); + } + + bool + is_finalized() { + return finalized; + } + + // Used to check kernel cache status (particularly after initialization) + error_t + status() { + if (get_status() != CUDNN_STATUS_SUCCESS) { + return {error_code_t::CUDNN_BACKEND_API_FAILED, + "CUDNN_BACKEND_KERNEL_CACHE_DESCRIPTOR: Check CUDNN_VERSION >= 9.4"}; + } + return {}; + } + + error_t + to_json(std::string &str_json) const { + str_json.clear(); +#if (CUDNN_VERSION >= 91000) + RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 91000, + error_code_t::CUDNN_BACKEND_API_FAILED, + "CUDNN_ATTR_KERNEL_CACHE_JSON_REPRESENTATION is only available starting 9.10."); + + int64_t serializationSize; + std::vector serialization_buf; + _CUDNN_CHECK_CUDNN_ERROR(detail::get_attribute( + get_ptr(), CUDNN_ATTR_KERNEL_CACHE_JSON_REPRESENTATION, CUDNN_TYPE_CHAR, 0, &serializationSize, nullptr)); + serialization_buf.resize(static_cast(serializationSize)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::get_attribute(get_ptr(), + CUDNN_ATTR_KERNEL_CACHE_JSON_REPRESENTATION, + CUDNN_TYPE_CHAR, + serializationSize, + &serializationSize, + serialization_buf.data())); + std::string json_string(serialization_buf.begin(), serialization_buf.end()); + str_json = std::move(json_string); + return {}; +#else + (void)str_json; + return {error_code_t::CUDNN_BACKEND_API_FAILED, + "CUDNN_ATTR_KERNEL_CACHE_JSON_REPRESENTATION is only available starting 9.10."}; +#endif + } + + error_t + from_json(const std::string &json_cache) { +#if (CUDNN_VERSION >= 91000) + RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 91000, + error_code_t::CUDNN_BACKEND_API_FAILED, + "CUDNN_ATTR_KERNEL_CACHE_JSON_REPRESENTATION is only available starting 9.10."); + + // Check if the kernel cache is already initialized + RETURN_CUDNN_FRONTEND_ERROR_IF( + get_ptr() != nullptr, error_code_t::CUDNN_BACKEND_API_FAILED, "Kernel cache is already initialized."); + + // // Initialize the kernel cache descriptor + CHECK_CUDNN_FRONTEND_ERROR(initialize(CUDNN_BACKEND_KERNEL_CACHE_DESCRIPTOR)); + + std::vector serialization_buf; + serialization_buf.assign(json_cache.begin(), json_cache.end()); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(get_ptr(), + CUDNN_ATTR_KERNEL_CACHE_JSON_REPRESENTATION, + CUDNN_TYPE_CHAR, + serialization_buf.size(), + serialization_buf.data())); + return {}; +#else + (void)json_cache; + return {error_code_t::CUDNN_BACKEND_API_FAILED, + "CUDNN_ATTR_KERNEL_CACHE_JSON_REPRESENTATION is only available starting 9.10."}; +#endif + } + + // Responsible for initializing, setting operation graph attribute, and finalizing kernel cache + // Check for both compile-time and runtime cuDNN version + error_t + build(cudnnBackendDescriptor_t op_graph) { +#if (CUDNN_VERSION >= 90400) + RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 90400, + error_code_t::GRAPH_NOT_SUPPORTED, + "CUDNN_BACKEND_KERNEL_CACHE_DESCRIPTOR is only available starting 9.4."); + if (get_ptr() == nullptr) { + CHECK_CUDNN_FRONTEND_ERROR(initialize(CUDNN_BACKEND_KERNEL_CACHE_DESCRIPTOR)); + } +#if (CUDNN_VERSION >= 90500) + RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 90500, + error_code_t::GRAPH_NOT_SUPPORTED, + "CUDNN_ATTR_KERNEL_CACHE_OPERATION_GRAPH is only available starting 9.5."); + if (op_graph) { + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute( + get_ptr(), CUDNN_ATTR_KERNEL_CACHE_OPERATION_GRAPH, CUDNN_TYPE_BACKEND_DESCRIPTOR, 1, &op_graph)); + } +#else + (void)op_graph; +#endif + CHECK_CUDNN_FRONTEND_ERROR(finalize()); + finalized = true; + return {}; +#else + (void)op_graph; + return {error_code_t::CUDNN_BACKEND_API_FAILED, + "CUDNN_BACKEND_KERNEL_CACHE_DESCRIPTOR is only available starting 9.4."}; +#endif + } + + private: + bool finalized = false; +}; +} // namespace cudnn_frontend \ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/backend/plan_helpers.h b/deps/cudnn-frontend/include/cudnn_frontend/backend/plan_helpers.h new file mode 100644 index 0000000..4ef47e2 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/backend/plan_helpers.h @@ -0,0 +1,212 @@ +#pragma once + +#include + +#include "cudnn.h" + +#include "backend_descriptor.h" +#include "../knobs.h" + +namespace cudnn_frontend::detail { +/** + * @brief Creates a CUDNN backend variant pack descriptor. + * + * This function creates a `backend_descriptor` object representing a CUDNN backend variant pack + * descriptor. The variant pack descriptor is configured with the provided device pointers, unique + * IDs, and a workspace pointer. + * + * @param[out] variant_pack The created `backend_descriptor` object representing the variant pack. + * @param device_ptrs A vector of device pointers to be associated with the variant pack. + * @param uids A vector of unique IDs to be associated with the variant pack. + * @param workspace_ptr A pointer to the workspace memory to be associated with the variant pack. + * @return `error_t` A tuple containing the error code and an optional error message. + * The error code is `error_code_t::OK` on success, or an appropriate error code on failure. + */ +inline error_t +get_workspace_size(ManagedOpaqueDescriptor& engine_config, int64_t& workspace) { +#if CUDNN_VERSION >= 90200 + _CUDNN_CHECK_CUDNN_ERROR(detail::get_attribute(engine_config->get_backend_descriptor(), + CUDNN_ATTR_ENGINECFG_WORKSPACE_SIZE, + CUDNN_TYPE_INT64, + 1, + nullptr, + &workspace)); + return {error_code_t::OK, ""}; +#else + (void)engine_config; + (void)workspace; + return {error_code_t::CUDNN_BACKEND_API_FAILED, + "CUDNN_ATTR_ENGINECFG_WORKSPACE_SIZE is only available starting 9.2."}; +#endif +} + +inline error_t +get_shared_memory_size(ManagedOpaqueDescriptor& engine_config, int32_t& shared_memory_size) { +#if CUDNN_VERSION >= 90200 + _CUDNN_CHECK_CUDNN_ERROR(detail::get_attribute(engine_config->get_backend_descriptor(), + CUDNN_ATTR_ENGINECFG_SHARED_MEMORY_USED, + CUDNN_TYPE_INT32, + 1, + nullptr, + &shared_memory_size)); + return {error_code_t::OK, ""}; +#else + (void)engine_config; + (void)shared_memory_size; + return {error_code_t::CUDNN_BACKEND_API_FAILED, + "CUDNN_ATTR_ENGINECFG_SHARED_MEMORY_USED is only available starting 9.2."}; +#endif +} + +inline error_t +create_engine(backend_descriptor& engine, + int64_t const engine_id, + cudnnBackendDescriptor_t op_graph, + std::shared_ptr device_properties = nullptr) { + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute( + engine.get_ptr(), CUDNN_ATTR_ENGINE_OPERATION_GRAPH, CUDNN_TYPE_BACKEND_DESCRIPTOR, 1, &op_graph)); + + // Validate before setting + int64_t count; + _CUDNN_CHECK_CUDNN_ERROR(detail::get_attribute( + op_graph, CUDNN_ATTR_OPERATIONGRAPH_ENGINE_GLOBAL_COUNT, CUDNN_TYPE_INT64, 1, nullptr, &count)); + RETURN_CUDNN_FRONTEND_ERROR_IF( + engine_id >= count || engine_id < 0, error_code_t::INVALID_VALUE, "Invalid engine id."); + + _CUDNN_CHECK_CUDNN_ERROR( + detail::set_attribute(engine.get_ptr(), CUDNN_ATTR_ENGINE_GLOBAL_INDEX, CUDNN_TYPE_INT64, 1, &engine_id)); + + if (device_properties != nullptr) { +#if (CUDNN_VERSION >= 90800) + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(engine.get_ptr(), + CUDNN_ATTR_ENGINE_DEVICEPROP, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &device_properties->get_ptr())); +#endif + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(engine.get_ptr())); + + return {error_code_t::OK, ""}; +} + +inline error_t +query_knobs(int64_t const engine_id, cudnnBackendDescriptor_t op_graph, std::vector& knobs) { + detail::backend_descriptor engine(CUDNN_BACKEND_ENGINE_DESCRIPTOR); + RETURN_CUDNN_FRONTEND_ERROR_IF(engine.get_status() != CUDNN_STATUS_SUCCESS, + error_code_t::CUDNN_BACKEND_API_FAILED, + "Failed to create engine's backend descriptor."); + CHECK_CUDNN_FRONTEND_ERROR(detail::create_engine(engine, engine_id, op_graph)); + + // Initialize a backend descriptor for each knob type + // The size of the array should be CUDNN_KNOB_TYPE_COUNTS, as currently we dont know how many knobs the engine will + // support + std::array frontend_knobs; + for (size_t i = 0; i < CUDNN_KNOB_TYPE_COUNTS; i++) { + backend_descriptor frontend_knob(CUDNN_BACKEND_KNOB_INFO_DESCRIPTOR); + RETURN_CUDNN_FRONTEND_ERROR_IF(frontend_knob.get_status() != CUDNN_STATUS_SUCCESS, + error_code_t::CUDNN_BACKEND_API_FAILED, + "Failed to create knob's backend descriptor."); + frontend_knobs[i] = std::move(frontend_knob); + } + + // Create an auxillary array to hold the raw knob descriptors + std::array backend_knobs; + for (size_t i = 0; i < CUDNN_KNOB_TYPE_COUNTS; i++) { + backend_knobs[i] = frontend_knobs[i].get_ptr(); + } + + // This is the actual number of knobs that is supported by the engine + int64_t knobs_size; + _CUDNN_CHECK_CUDNN_ERROR(detail::get_attribute(engine.get_ptr(), + CUDNN_ATTR_ENGINE_KNOB_INFO, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + CUDNN_KNOB_TYPE_COUNTS, + &knobs_size, + backend_knobs.data())); + + for (int64_t i = 0; i < knobs_size; i++) { + cudnnBackendKnobType_t type; + int64_t elemCount; + _CUDNN_CHECK_CUDNN_ERROR(detail::get_attribute( + frontend_knobs[i].get_ptr(), CUDNN_ATTR_KNOB_INFO_TYPE, CUDNN_TYPE_KNOB_TYPE, 1, &elemCount, &type)); + + int64_t maxValue; + _CUDNN_CHECK_CUDNN_ERROR(detail::get_attribute(frontend_knobs[i].get_ptr(), + CUDNN_ATTR_KNOB_INFO_MAXIMUM_VALUE, + CUDNN_TYPE_INT64, + 1, + &elemCount, + &maxValue)); + + int64_t minValue; + _CUDNN_CHECK_CUDNN_ERROR(detail::get_attribute(frontend_knobs[i].get_ptr(), + CUDNN_ATTR_KNOB_INFO_MINIMUM_VALUE, + CUDNN_TYPE_INT64, + 1, + &elemCount, + &minValue)); + + int64_t stride; + _CUDNN_CHECK_CUDNN_ERROR(detail::get_attribute( + frontend_knobs[i].get_ptr(), CUDNN_ATTR_KNOB_INFO_STRIDE, CUDNN_TYPE_INT64, 1, &elemCount, &stride)); + + auto frontend_knob_type = convert_from_backend_knob_type(type); + knobs.emplace_back(frontend_knob_type, maxValue, minValue, stride); + } + + return {error_code_t::OK, ""}; +} + +inline error_t +set_knob_choices(std::unordered_map const& user_choices, + std::vector& knob_choices) { + for (auto const& [type, choice] : user_choices) { + backend_descriptor knob_choice(CUDNN_BACKEND_KNOB_CHOICE_DESCRIPTOR); + RETURN_CUDNN_FRONTEND_ERROR_IF(knob_choice.get_status() != CUDNN_STATUS_SUCCESS, + error_code_t::CUDNN_BACKEND_API_FAILED, + "Failed to create knob_choice's backend descriptor."); + + cudnnBackendKnobType_t backend_type; + _CUDNN_CHECK_CUDNN_ERROR(convert_to_backend_knob_type(type, backend_type)); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute( + knob_choice.get_ptr(), CUDNN_ATTR_KNOB_CHOICE_KNOB_TYPE, CUDNN_TYPE_KNOB_TYPE, 1, &backend_type)); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute( + knob_choice.get_ptr(), CUDNN_ATTR_KNOB_CHOICE_KNOB_VALUE, CUDNN_TYPE_INT64, 1, &choice)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(knob_choice.get_ptr())); + + knob_choices.push_back(std::move(knob_choice)); + } + + return {error_code_t::OK, ""}; +} + +inline error_t +create_engine_config(ManagedOpaqueDescriptor& engine_config, + backend_descriptor& engine, + std::vector& knob_choices) { + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(engine_config->get_backend_descriptor(), + CUDNN_ATTR_ENGINECFG_ENGINE, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &(engine.get_ptr()))); + + std::vector backend_knob_choices(CUDNN_KNOB_TYPE_COUNTS); + for (size_t i = 0; i < knob_choices.size(); i++) { + backend_knob_choices[i] = knob_choices[i].get_ptr(); + } + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(engine_config->get_backend_descriptor(), + CUDNN_ATTR_ENGINECFG_KNOB_CHOICES, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + knob_choices.size(), + backend_knob_choices.data())); + + // Finalizing the descriptor + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(engine_config->get_backend_descriptor())); + + return {error_code_t::OK, ""}; +} + +} // namespace cudnn_frontend::detail diff --git a/deps/cudnn-frontend/include/cudnn_frontend/context.h b/deps/cudnn-frontend/include/cudnn_frontend/context.h new file mode 100644 index 0000000..86e878d --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/context.h @@ -0,0 +1,138 @@ +#pragma once + +#include "../cudnn_frontend_shim.h" +#include "../cudnn_frontend_utils.h" + +namespace cudnn_frontend::detail { + +class Context { + DataType_t compute_data_type = DataType_t::NOT_SET; + DataType_t intermediate_data_type = DataType_t::NOT_SET; + DataType_t io_data_type = DataType_t::NOT_SET; + int32_t target_sm_count = -1; + mutable int32_t target_sm_version = -1; + bool is_dynamic_shape_enabled = false; + bool is_override_shape_enabled = false; + + std::string name = ""; + + public: + Context& + set_intermediate_data_type(DataType_t const type) { + intermediate_data_type = type; + return *this; + } + + Context& + set_io_data_type(DataType_t const type) { + io_data_type = type; + return *this; + } + + Context& + set_compute_data_type(DataType_t const type) { + compute_data_type = type; + return *this; + } + + DataType_t + get_io_data_type() const { + return io_data_type; + } + + DataType_t + get_intermediate_data_type() const { + return intermediate_data_type; + } + + DataType_t + get_compute_data_type() const { + return compute_data_type; + } + + Context& + set_name(std::string const& name_) { + name = name_; + return *this; + } + + std::string + get_name() const { + return name; + } + + Context& + set_target_sm_count(int32_t count) { + target_sm_count = count; + return *this; + } + + Context& + set_sm_version(int32_t version) { + target_sm_version = version; + return *this; + } + + Context& + set_dynamic_shape_enabled(bool is_enabled) { + is_dynamic_shape_enabled = is_enabled; + return *this; + } + + Context& + set_override_shape_enabled(bool is_enabled) { + is_override_shape_enabled = is_enabled; + return *this; + } + + bool + get_dynamic_shape_enabled() const { + return is_dynamic_shape_enabled; + } + + bool + get_override_shape_enabled() const { + return is_override_shape_enabled; + } + + int32_t + get_target_sm_count() const { + return target_sm_count; + } + + int32_t + get_sm_version() const { + return target_sm_version; + } + + error_t + populate_sm_version_from_device() const { + if (target_sm_version > 0) { + // Already set by user or previous call + return {error_code_t::OK, ""}; + } + cudaDeviceProp prop; + int device; + _CUDNN_CHECK_CUDA_ERROR(cuda_get_device(&device)); + _CUDNN_CHECK_CUDA_ERROR(cuda_get_device_properties(&prop, device)); + target_sm_version = prop.major * 10 + prop.minor; + CUDNN_FE_LOG_LABEL_ENDL("INFO: Populated SM version from device: " << device << " " << target_sm_version); + return {error_code_t::OK, ""}; + } + + Context& + fill_missing_properties(Context const& global_context) { + if (get_compute_data_type() == DataType_t::NOT_SET) { + set_compute_data_type(global_context.get_compute_data_type()); + } + if (get_intermediate_data_type() == DataType_t::NOT_SET) { + set_intermediate_data_type(global_context.get_intermediate_data_type()); + } + if (get_io_data_type() == DataType_t::NOT_SET) { + set_io_data_type(global_context.get_io_data_type()); + } + return *this; + } +}; + +} // namespace cudnn_frontend::detail \ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/cudnn_interface.h b/deps/cudnn-frontend/include/cudnn_frontend/cudnn_interface.h new file mode 100644 index 0000000..c16d6eb --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/cudnn_interface.h @@ -0,0 +1,251 @@ +#pragma once + +#include +#include +#include + +#include "../cudnn_frontend_Tensor.h" +#include "../cudnn_frontend_Operation.h" +#include "../cudnn_frontend_OperationGraph.h" +#include "../cudnn_frontend_EngineConfig.h" +#include "../cudnn_frontend_ExecutionPlan.h" +#include "../cudnn_frontend_VariantPack.h" + +#include "graph_properties.h" +#include "graph_helpers.h" +#include "plans.h" + +namespace cudnn_frontend { + +namespace detail { +inline void +assign_uid(graph::Tensor_attributes* const tensor, + int64_t& potential_uid, + std::unordered_set const& used_uids) { + // get_next_potential_uid + while (used_uids.find(potential_uid) != used_uids.end()) { + ++potential_uid; + } + + tensor->set_uid(potential_uid); + ++potential_uid; // increment, as used its used now +} + +// TODO: Always returns OK. Can the status and error message be accessed from tensor descriptor? +inline error_t +create_cudnn_tensor( + std::shared_ptr const& props, + std::unordered_map>& tensors, + int64_t& potential_uid, + std::unordered_set const& used_uids) { + // Assign tensor a uid + if (props->has_uid() == false) { + assign_uid(props.get(), potential_uid, used_uids); + } + + // Check whether backend tensor already created + auto tensor_uid = props->get_uid(); + if (tensors.find(tensor_uid) != tensors.end()) { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Backend Tensor named '" << props->get_name() << "' with UID " << tensor_uid + << " already created."); + return {error_code_t::OK, ""}; + } + CUDNN_FE_LOG_LABEL_ENDL("INFO: Backend Tensor named '" << props->get_name() << "' with UID " << tensor_uid + << " being created."); + + auto&& tensor_builder = cudnn_frontend::TensorBuilder(); + + tensor_builder.setDim(props->get_dim().size(), props->get_dim().data()) + .setStrides(props->get_stride().size(), props->get_stride().data()) + .setId(tensor_uid) + .setAlignment(props->get_alignment()) + .setDataType(props->get_data_type()) + .setVirtual(props->get_is_virtual()) + .setByValue(props->get_is_pass_by_value()) + .setReorderType(props->get_reordering_type()); + + // Set vector count and dimension if they are non-default + if (props->get_vector_count() > 1 || props->get_vector_dimension() >= 0) { + tensor_builder.setVectorCountAndDimension(props->get_vector_count(), props->get_vector_dimension()); + } + + // Set compile-time constant value before build (if present) + if (props->get_has_compile_time_constant()) { + auto const_value = props->get_compile_time_constant(); + if (const_value.has_value()) { + std::visit([&tensor_builder](auto&& val) { tensor_builder.setConstValue(val); }, *const_value); + + CUDNN_FE_LOG_LABEL_ENDL("INFO: Compile-time constant value set for tensor '" << props->get_name() + << "'"); + } + } + + if (auto ragged_offset_props = props->get_ragged_offset()) { + CHECK_CUDNN_FRONTEND_ERROR(create_cudnn_tensor(ragged_offset_props, tensors, potential_uid, used_uids)); + tensor_builder.setRaggedOffset(tensors.at(ragged_offset_props->get_uid())); + } + if (props->has_ragged_offset_multiplier()) { + tensor_builder.setRaggedOffsetMultiplier(props->get_ragged_offset_multiplier()); + } + +#ifdef NV_CUDNN_DISABLE_EXCEPTION + // disable exception macro is defined. Calling build will not throw. + // Check status of desc and return error. + auto tensor = tensor_builder.build(); + RETURN_CUDNN_FRONTEND_ERROR_IF( + tensor.get_status() != CUDNN_STATUS_SUCCESS, error_code_t::CUDNN_BACKEND_API_FAILED, tensor.get_error()); + tensors.emplace(tensor_uid, std::make_shared(std::move(tensor))); +#else + // build() can throw + // wrap in try catch + try { + auto tensor = tensor_builder.build(); + tensors.emplace(tensor_uid, std::make_shared(std::move(tensor))); + } catch (cudnn_frontend::cudnnException& e) { + RETURN_CUDNN_FRONTEND_ERROR_IF( + e.getCudnnStatus() != CUDNN_STATUS_SUCCESS, error_code_t::CUDNN_BACKEND_API_FAILED, e.what()); + } +#endif + + return {error_code_t::OK, ""}; +} +} // namespace detail + +namespace graph { +class UnifiedSDPANode; +} // namespace graph + +class ICudnn { + friend class graph::UnifiedSDPANode; + + protected: + using uid_t = int64_t; + + //// Store tensors and operations as they (probably?) need to be kept alive. + // + // The tensor mapping from fe::Tensor to be::Tensor. + // + // sub nodes share fe::Tensor. Example, in a conv-bias graph, conv output Y and bias input IN_0 are the same + // fe::Tensor. But both sub ndoes need to work together to make sure only one be::Tensor is created. Hence this + // uid_to_backend_tensors acts as the global registry for each sub node to use. + // + // Key cannot be fe::Tensor, or shared_ptr, or underlying object address of fe::Tensor. + // Hence using uid, as that uniquely identifies both types of tensors. + std::unordered_map> uid_to_tensors; + std::vector> operations; + graph::managed_backend_descriptor_t raw_operations; + + std::shared_ptr operation_graph; + std::unordered_set variant_pack_uids; + + graph::Execution_plan_list plans; + + bool is_dynamic_shape_enabled = false; + std::shared_ptr kernel_cache = nullptr; + + bool is_override_shape_enabled = false; + + std::shared_ptr device_properties = nullptr; + + std::shared_ptr + get_operation_graph() const { + return operation_graph; + } + + error_t + create_cudnn_operation_graph(cudnnHandle_t handle) { + std::vector cudnn_operations; + for (std::shared_ptr operation : operations) { + cudnn_operations.push_back(operation.get()); + } + + auto&& cudnn_operation_graph_builder = cudnn_frontend::OperationGraphBuilder(); + cudnn_operation_graph_builder.setHandle(handle) + .setOperationGraph(cudnn_operations.size(), cudnn_operations.data()) + .setIsDynamicShapeEnabled(is_dynamic_shape_enabled) + .setIsOverrideShapeEnabled(is_override_shape_enabled); + for (auto& op : raw_operations) { + cudnn_operation_graph_builder.addOperation(op); + } + +#ifdef NV_CUDNN_DISABLE_EXCEPTION + // disable exception macro is defined. Calling build will not throw. + // Check status of desc and return error. + auto cudnn_operation_graph = cudnn_operation_graph_builder.build(); + RETURN_CUDNN_FRONTEND_ERROR_IF(cudnn_operation_graph.get_status() != CUDNN_STATUS_SUCCESS, + error_code_t::CUDNN_BACKEND_API_FAILED, + cudnn_operation_graph.get_error()); + operation_graph = std::make_shared(std::move(cudnn_operation_graph)); +#else + // build() can throw + // wrap in try catch + try { + auto cudnn_operation_graph = cudnn_operation_graph_builder.build(); + operation_graph = std::make_shared(std::move(cudnn_operation_graph)); + } catch (cudnn_frontend::cudnnException& e) { + RETURN_CUDNN_FRONTEND_ERROR_IF( + e.getCudnnStatus() != CUDNN_STATUS_SUCCESS, error_code_t::CUDNN_BACKEND_API_FAILED, e.what()); + } +#endif + return {error_code_t::OK, "Successfully built Operation Graph."}; + } + + public: + error_t + get_cudnn_workspace_size_node(int64_t const plan_index, int64_t& cudnn_workspace_size) const { + CHECK_CUDNN_FRONTEND_ERROR(plans.is_plan_index_executable(plan_index)); + + cudnn_workspace_size = std::max(cudnn_workspace_size, plans.execution_plans[plan_index]->getWorkspaceSize()); + + return {error_code_t::OK, ""}; + } + + int64_t + get_max_cudnn_workspace_size_node() const { + return plans.get_autotune_workspace(); + } + + error_t + execute_cudnn_plan_with_uid(cudnnHandle_t handle, + std::unordered_map const& tensor_uid_to_pointer_map, + void* workspace_ptr, + int64_t plan_index, + std::vector const& override_uids, + std::vector> const& override_shapes, + std::vector> const& override_strides) const { + // Make sure device pointer is provided for all uids expected for this plan + std::vector device_ptrs; + std::vector uids; + for (auto const& uid : variant_pack_uids) { + auto search = tensor_uid_to_pointer_map.find(uid); + RETURN_CUDNN_FRONTEND_ERROR_IF(search == tensor_uid_to_pointer_map.end(), + error_code_t::INVALID_VARIANT_PACK, + "Uid " + std::to_string(uid) + " does not exist in variant pack."); + device_ptrs.push_back(search->second); + uids.push_back(uid); + } + + CHECK_CUDNN_FRONTEND_ERROR(plans.is_plan_index_executable(plan_index)); + + CUDNN_FE_LOG_LABEL_ENDL("INFO: Executing plan at index " << plan_index + << " with override uids: " << override_uids.size()); + + if (override_uids.size() == 0) { + CHECK_CUDNN_FRONTEND_ERROR( + detail::execute(handle, plans.execution_plans[plan_index].get(), device_ptrs, uids, workspace_ptr)); + } else { + CHECK_CUDNN_FRONTEND_ERROR(detail::execute(handle, + plans.execution_plans[plan_index].get(), + device_ptrs, + uids, + workspace_ptr, + override_uids, + override_shapes, + override_strides)); + } + + return {error_code_t::OK, ""}; + } +}; + +} // namespace cudnn_frontend diff --git a/deps/cudnn-frontend/include/cudnn_frontend/experimental/attention_utils.h b/deps/cudnn-frontend/include/cudnn_frontend/experimental/attention_utils.h new file mode 100644 index 0000000..b44a55e --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/experimental/attention_utils.h @@ -0,0 +1,385 @@ +#pragma once + +#include +#include +#include +#include + +// cudaLibrary_t, cudaKernel_t, cudaJitOption, cudaLibraryOption etc. +// were introduced in CUDA 12.8. On older toolkits (or when using dynamic +// loading where cuda_runtime.h may lack these types), provide typedefs +// from the driver API equivalents (available via cuda.h) so that +// engine headers can declare members without #ifdef clutter. +// The actual runtime API functions are guarded by CUDART_VERSION checks below. +#if !defined(CUDART_VERSION) || CUDART_VERSION < 12080 +using cudaLibrary_t = CUlibrary; +using cudaKernel_t = CUkernel; +using cudaJitOption = CUjit_option; +using cudaLibraryOption = CUlibraryOption; +#endif + +namespace cudnn_frontend::experimental { + +// ============================================================ +// Internal types matching generated kernel parameter layouts +// ============================================================ + +struct AttentionShape_t { + uint32_t b, q_h, k_h, v_h, s_q, s_kv, d_qk, d_v; +}; + +struct AttentionDescriptor_t { + uint32_t b, q_h, k_h, v_h, s_q, s_kv, d_qk, d_v; + uint16_t q_heads_per_k, q_heads_per_v, min_q_heads_per_kv; +}; + +struct FastDivisor_t { + uint32_t val, shr, mul; +}; + +struct tensor_descriptor { + static const int MAX_DIMS = 12; + int64_t num_dims; + int64_t dims[MAX_DIMS]; + int64_t strides[MAX_DIMS]; +}; + +// ============================================================ +// Utility functions +// ============================================================ + +inline int +div_up(int a, int b) { + return (a + b - 1) / b; +} + +// floor(log2(x)) for x > 0. +inline int +find_log_2_floor(uint32_t x) { + if (x <= 1) return 0; + int a = 0; + while ((1u << (a + 1)) <= x) a++; + return a; +} + +// Compute FastDivisor_t for the kernel's fastDivMod which uses: +// div = __umulhi(2 * val, mul) >> shr +// This matches cuDNN backend's find_divisor_v2 fast-math helper. +inline FastDivisor_t +make_fast_divisor(uint32_t divisor) { + FastDivisor_t d; + d.val = divisor; + + if (divisor <= 1) { + // Division by 1: umulhi(2*val, 0x80000000) >> 0 = val, mod = 0 + d.shr = 0; + d.mul = 0x80000000u; + return d; + } + + // find_log_2(2 * divisor, round_up=true) + uint32_t x2 = 2u * divisor; + int a = 0; + { + uint32_t tmp = x2; + while (tmp > 1) { + tmp >>= 1; + a++; + } + } + // round up if not a power of 2 + if (x2 & (x2 - 1)) a++; + + uint32_t p = 31 + static_cast(a); + d.mul = static_cast(((1ULL << p) + static_cast(x2) - 1) / static_cast(x2)); + d.shr = p - 32; + return d; +} + +inline std::vector +parse_flags_string(const char* data, size_t len) { + std::vector flags; + std::string content(data, len); + std::istringstream stream(content); + std::string line; + while (std::getline(stream, line)) { + size_t start = line.find_first_not_of(" \t\r\n"); + if (start == std::string::npos) continue; + size_t end = line.find_last_not_of(" \t\r\n"); + line = line.substr(start, end - start + 1); + if (!line.empty()) { + flags.push_back(line); + } + } + return flags; +} + +// ============================================================ +// Kernel specification (compile-time metadata per kernel variant) +// ============================================================ + +struct KernelSpec { + const char* source; + size_t source_len; + const char* flags_raw; + size_t flags_len; + const char* kernel_name; + int tile_m, tile_n, tile_k; + int smem_bytes; +}; + +// ============================================================ +// CUDA runtime API wrappers (using NV_FE_CALL_TO_CUDA) +// ============================================================ + +namespace detail { + +// NV_FE_CALL_TO_CUDA macros reference symbols in cudnn_frontend::detail +// via unqualified lookup. Import them into this namespace. +#if defined NV_CUDNN_FRONTEND_USE_DYNAMIC_LOADING +using cudnn_frontend::detail::CudaLibrary; +using cudnn_frontend::detail::get_cuda_symbol; +#endif + +// Re-export CUDA runtime wrappers from the main shim so the engine +// never calls cudaMalloc/cudaFree/etc. directly (required for dynamic loading). +using cudnn_frontend::detail::cuda_get_device; +using cudnn_frontend::detail::cuda_get_device_properties; +using cudnn_frontend::detail::cuda_get_error_string; +using cudnn_frontend::detail::cuda_mem_cpy_async; +using cudnn_frontend::detail::cuda_mem_set_async; + +// Write a 32-bit pattern to device memory (async, on stream). +// Uses thread_local storage so the source buffer persists through the async copy. +// Only used with N=1 in practice (writing float 1.0f for FP8 default scale). +// Cannot use NV_FE_CALL_TO_CUDA because cudaMemcpyAsync has more args than this wrapper. +inline cudaError_t +cuda_mem_set_d32_async(void* dstDevice, unsigned int ui, size_t N, cudaStream_t stream) { + static thread_local unsigned int val; + val = ui; +#if defined NV_CUDNN_FRONTEND_USE_DYNAMIC_LOADING + using fn_t = cudaError_t (*)(void*, const void*, size_t, cudaMemcpyKind, cudaStream_t); + auto _fn = reinterpret_cast(cudnn_frontend::detail::get_cuda_symbol(CudaLibrary::CUDART, "cudaMemcpyAsync")); + return _fn(dstDevice, &val, N * sizeof(unsigned int), cudaMemcpyHostToDevice, stream); +#else + return cudaMemcpyAsync(dstDevice, &val, N * sizeof(unsigned int), cudaMemcpyHostToDevice, stream); +#endif +} + +// Convert cudaError_t to a descriptive string (e.g., "invalid argument") +inline std::string +cuda_error_to_string(cudaError_t err) { + const char* str = cuda_get_error_string(err); + return str ? std::string(str) : ("cudaError=" + std::to_string(static_cast(err))); +} + +// cudaGetLastError takes zero arguments — can't use NV_FE_CALL_TO_CUDA +// (variadic macro requires at least one arg). Handle both paths manually. +inline cudaError_t +cuda_get_last_error() { +#if defined NV_CUDNN_FRONTEND_USE_DYNAMIC_LOADING + using fn_t = cudaError_t (*)(); + auto _fn = reinterpret_cast(cudnn_frontend::detail::get_cuda_symbol(CudaLibrary::CUDART, "cudaGetLastError")); + return _fn(); +#else + return cudaGetLastError(); +#endif +} + +// ============================================================ +// CUDA 12.8+ runtime API wrappers for library/kernel management. +// These APIs (cudaLibraryLoadData, cudaLibraryGetKernel, cudaLibraryUnload, +// cudaKernelSetAttributeForDevice, cudaGetDriverEntryPointByVersion) require +// CUDART_VERSION >= 12080. On older toolkits, the OSS engine check_support() +// will reject the configuration before these are called. +// ============================================================ + +#if !defined(NV_CUDNN_FRONTEND_USE_DYNAMIC_LOADING) && defined(CUDART_VERSION) && CUDART_VERSION < 12080 + +// Stubs that return errors — OSS engines require CUDA 12.8+. +inline cudaError_t +cuda_library_load_data(void*, const void*, void*, void**, unsigned int, void*, void**, unsigned int) { + return cudaErrorNotSupported; +} +inline cudaError_t +cuda_library_get_kernel(void*, void*, const char*) { + return cudaErrorNotSupported; +} +inline cudaError_t +cuda_library_unload(void*) { + return cudaErrorNotSupported; +} +inline cudaError_t +cuda_kernel_set_attribute_for_device(void*, int, int, int) { + return cudaErrorNotSupported; +} + +#else // CUDART_VERSION >= 12080 or dynamic loading + +inline cudaError_t +cuda_library_load_data(cudaLibrary_t* library, + const void* code, + cudaJitOption* jitOptions, + void** jitOptionsValues, + unsigned int numJitOptions, + cudaLibraryOption* libraryOptions, + void** libraryOptionsValues, + unsigned int numLibraryOptions) { + NV_FE_CALL_TO_CUDA(cuda_library_load_data, + cudaLibraryLoadData, + library, + code, + jitOptions, + jitOptionsValues, + numJitOptions, + libraryOptions, + libraryOptionsValues, + numLibraryOptions); +} + +inline cudaError_t +cuda_library_get_kernel(cudaKernel_t* pKernel, cudaLibrary_t library, const char* name) { + NV_FE_CALL_TO_CUDA(cuda_library_get_kernel, cudaLibraryGetKernel, pKernel, library, name); +} + +inline cudaError_t +cuda_library_unload(cudaLibrary_t library) { + NV_FE_CALL_TO_CUDA(cuda_library_unload, cudaLibraryUnload, library); +} + +inline cudaError_t +cuda_kernel_set_attribute_for_device(cudaKernel_t kernel, cudaFuncAttribute attrib, int val, int dev) { + NV_FE_CALL_TO_CUDA(cuda_kernel_set_attribute_for_device, cudaKernelSetAttributeForDevice, kernel, attrib, val, dev); +} + +#endif // CUDART_VERSION check + +inline cudaError_t +cuda_launch_kernel(const void* func, dim3 gridDim, dim3 blockDim, void** args, size_t sharedMem, cudaStream_t stream) { + NV_FE_CALL_TO_CUDA(cuda_launch_kernel, cudaLaunchKernel, func, gridDim, blockDim, args, sharedMem, stream); +} + +// cuTensorMapEncodeTiled has no runtime API equivalent. +// Resolve the driver function pointer via cudaGetDriverEntryPointByVersion +// so we never link against libcuda.so directly. Requires CUDA 12.8+. +inline CUresult +cu_tensor_map_encode_tiled(CUtensorMap* tensorMap, + CUtensorMapDataType tensorDataType, + cuuint32_t tensorRank, + void* globalAddress, + const cuuint64_t* globalDim, + const cuuint64_t* globalStrides, + const cuuint32_t* boxDim, + const cuuint32_t* elementStrides, + CUtensorMapInterleave interleave, + CUtensorMapSwizzle swizzle, + CUtensorMapL2promotion l2Promotion, + CUtensorMapFloatOOBfill oobFill) { +#if !defined(NV_CUDNN_FRONTEND_USE_DYNAMIC_LOADING) && defined(CUDART_VERSION) && CUDART_VERSION < 12080 + (void)tensorMap; + (void)tensorDataType; + (void)tensorRank; + (void)globalAddress; + (void)globalDim; + (void)globalStrides; + (void)boxDim; + (void)elementStrides; + (void)interleave; + (void)swizzle; + (void)l2Promotion; + (void)oobFill; + return CUDA_ERROR_NOT_SUPPORTED; +#else + using PFN = CUresult(CUDAAPI*)(CUtensorMap*, + CUtensorMapDataType, + cuuint32_t, + void*, + const cuuint64_t*, + const cuuint64_t*, + const cuuint32_t*, + const cuuint32_t*, + CUtensorMapInterleave, + CUtensorMapSwizzle, + CUtensorMapL2promotion, + CUtensorMapFloatOOBfill); + // Thread-safe static initialization (C++11 guarantees) + static const PFN pfn = []() -> PFN { + void* raw_pfn = nullptr; + cudaDriverEntryPointQueryResult status; +#if defined NV_CUDNN_FRONTEND_USE_DYNAMIC_LOADING + // In dynamic loading mode, resolve cudaGetDriverEntryPointByVersion via dlsym + using GetEntryPointFn = + cudaError_t (*)(const char*, void**, unsigned int, int, cudaDriverEntryPointQueryResult*); + auto get_entry_point = reinterpret_cast( + cudnn_frontend::detail::get_cuda_symbol(CudaLibrary::CUDART, "cudaGetDriverEntryPointByVersion")); + cudaError_t err = get_entry_point("cuTensorMapEncodeTiled", &raw_pfn, 12000, cudaEnableDefault, &status); +#else + cudaError_t err = + cudaGetDriverEntryPointByVersion("cuTensorMapEncodeTiled", &raw_pfn, 12000, cudaEnableDefault, &status); +#endif + if (err != cudaSuccess || status != cudaDriverEntryPointSuccess || !raw_pfn) { + return nullptr; + } + return reinterpret_cast(raw_pfn); + }(); + if (!pfn) { + return CUDA_ERROR_NOT_FOUND; + } + return pfn(tensorMap, + tensorDataType, + tensorRank, + globalAddress, + globalDim, + globalStrides, + boxDim, + elementStrides, + interleave, + swizzle, + l2Promotion, + oobFill); +#endif // CUDART_VERSION check +} + +} // namespace detail + +// ============================================================ +// TMA descriptor creation (4D) +// ============================================================ +inline error_t +create_tma_desc_4d(CUtensorMap* desc, + void* globalAddress, + CUtensorMapDataType dataType, + uint32_t dim0, + uint32_t dim1, + uint32_t dim2, + uint32_t dim3, + uint64_t stride1_bytes, + uint64_t stride2_bytes, + uint64_t stride3_bytes, + uint32_t boxDim0, + uint32_t boxDim1, + CUtensorMapSwizzle swizzle) { + uint64_t globalDims[4] = {dim0, dim1, dim2, dim3}; + uint64_t globalStrides[3] = {stride1_bytes, stride2_bytes, stride3_bytes}; + uint32_t boxDims[4] = {boxDim0, boxDim1, 1, 1}; + uint32_t elemStrides[4] = {1, 1, 1, 1}; + + CUresult err_status = detail::cu_tensor_map_encode_tiled(desc, + dataType, + 4, + globalAddress, + globalDims, + globalStrides, + boxDims, + elemStrides, + CU_TENSOR_MAP_INTERLEAVE_NONE, + swizzle, + CU_TENSOR_MAP_L2_PROMOTION_NONE, + CU_TENSOR_MAP_FLOAT_OOB_FILL_NAN_REQUEST_ZERO_FMA); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + err_status != CUDA_SUCCESS, error_code_t::CUDA_API_FAILED, "cuTensorMapEncodeTiled failed"); + return {error_code_t::OK, ""}; +} + +} // namespace cudnn_frontend::experimental \ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/experimental/nvrtc_shim.h b/deps/cudnn-frontend/include/cudnn_frontend/experimental/nvrtc_shim.h new file mode 100644 index 0000000..eaad3de --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/experimental/nvrtc_shim.h @@ -0,0 +1,252 @@ +/* + * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#pragma once + +#include +#include // for cudaError_t (needed by cudnn_frontend_shim.h wrappers) + +// Pull in HMODULE, dlopen, dlsym, dlclose macros and platform definitions +// from the existing cudnn_frontend shim infrastructure. +#include "../../cudnn_frontend_shim.h" + +#if defined NV_CUDNN_FRONTEND_USE_DYNAMIC_LOADING +#include +#endif + +namespace cudnn_frontend { +namespace experimental { +namespace detail { + +#if defined NV_CUDNN_FRONTEND_USE_DYNAMIC_LOADING + +/// @brief Attempt to dynamically load the NVRTC shared library. +/// +/// Tries a sequence of candidate library names in order, returning +/// the handle for the first one that loads successfully. +/// +/// @return A valid library handle on success, or nullptr on failure. +/// This function does NOT throw. +inline HMODULE +load_nvrtc_so() { +#ifdef _WIN32 + constexpr const char *candidates[] = { + "nvrtc64_131_0.dll", + "nvrtc64_130_0.dll", + "nvrtc64_120_0.dll", + }; +#else + constexpr const char *candidates[] = { + "libnvrtc.so", + "libnvrtc.so.13", + "libnvrtc.so.12", + }; +#endif + constexpr size_t num_candidates = sizeof(candidates) / sizeof(candidates[0]); + + for (size_t i = 0; i < num_candidates; ++i) { + // Clear any prior error state + dlerror(); + + HMODULE handle = dlopen(candidates[i], RTLD_NOW); + if (handle) { + return handle; + } + } + + // All candidates failed -- return nullptr (no throw) + return nullptr; +} + +/// @brief Look up a symbol in the dynamically-loaded NVRTC library. +/// +/// The library handle is loaded once (lazily, thread-safe) and cached +/// in a function-local static. If the library could not be loaded, +/// every call returns nullptr. +/// +/// @param function_name The mangled / C symbol name to resolve. +/// @return A generic pointer to the symbol, or nullptr if the library +/// is not loaded or the symbol is not found. +inline void * +get_nvrtc_symbol(const char *function_name) { + static std::mutex nvrtc_lib_mutex; + static HMODULE nvrtc_handle = nullptr; + static bool load_attempted = false; + + std::lock_guard lock(nvrtc_lib_mutex); + + if (!load_attempted) { + load_attempted = true; + nvrtc_handle = load_nvrtc_so(); + } + + if (!nvrtc_handle) { + return nullptr; + } + + // Clear any existing error before calling dlsym + dlerror(); + + void *symbol = dlsym(nvrtc_handle, function_name); + // If dlsym fails we simply return nullptr -- callers decide how to + // report the error (the macro returns NVRTC_ERROR_INTERNAL_ERROR). + return symbol; +} + +/// @brief Check whether the NVRTC library was successfully loaded. +/// +/// Internally triggers a lazy load attempt if one has not been made yet. +/// +/// @return true if the library handle is valid, false otherwise. +inline bool +nvrtc_is_loaded() { + // Trigger the lazy load by requesting any symbol (the name is + // irrelevant -- we only care about whether the handle is valid). + // get_nvrtc_symbol will initialise the handle on the first call. + get_nvrtc_symbol("nvrtcVersion"); + + // We cannot directly inspect the static inside get_nvrtc_symbol, + // so we probe for a symbol that must exist in every NVRTC build. + return get_nvrtc_symbol("nvrtcVersion") != nullptr; +} + +/// @brief Macro for calling an NVRTC function via dynamic loading. +/// +/// Looks up @p nvrtc_symbol by name, and if the symbol is nullptr +/// returns @c NVRTC_ERROR_INTERNAL_ERROR. Otherwise casts the +/// pointer to the wrapper function's own signature and tail-calls it. +#define NV_FE_CALL_TO_NVRTC(function_name, nvrtc_symbol, ...) \ + void *fptr = get_nvrtc_symbol(#nvrtc_symbol); \ + if (fptr == nullptr) { \ + return NVRTC_ERROR_INTERNAL_ERROR; \ + } \ + return reinterpret_cast(fptr)(__VA_ARGS__); + +#else // static linking + +/// @brief When statically linked NVRTC is always considered loaded. +inline bool +nvrtc_is_loaded() { + return true; +} + +/// @brief Macro for calling an NVRTC function via static linking. +/// +/// Simply forwards the call directly to the NVRTC symbol. +#define NV_FE_CALL_TO_NVRTC(function_name, nvrtc_symbol, ...) return nvrtc_symbol(__VA_ARGS__); + +#endif // NV_CUDNN_FRONTEND_USE_DYNAMIC_LOADING + +// --------------------------------------------------------------------------- +// Wrapper functions for the NVRTC API +// +// Each wrapper has the exact same signature as the underlying NVRTC +// function it delegates to. Under dynamic loading the macro resolves +// the symbol at first call; under static linking the call is direct. +// --------------------------------------------------------------------------- + +/// @brief Create an NVRTC program from source. +/// @see nvrtcCreateProgram +inline nvrtcResult +nvrtc_create_program(nvrtcProgram *prog, + const char *src, + const char *name, + int numHeaders, + const char *const *headers, + const char *const *includeNames) { + NV_FE_CALL_TO_NVRTC(nvrtc_create_program, nvrtcCreateProgram, prog, src, name, numHeaders, headers, includeNames); +} + +/// @brief Destroy an NVRTC program. +/// @see nvrtcDestroyProgram +inline nvrtcResult +nvrtc_destroy_program(nvrtcProgram *prog) { + NV_FE_CALL_TO_NVRTC(nvrtc_destroy_program, nvrtcDestroyProgram, prog); +} + +/// @brief Compile an NVRTC program with the given options. +/// @see nvrtcCompileProgram +inline nvrtcResult +nvrtc_compile_program(nvrtcProgram prog, int numOptions, const char *const *options) { + NV_FE_CALL_TO_NVRTC(nvrtc_compile_program, nvrtcCompileProgram, prog, numOptions, options); +} + +/// @brief Query the size of the compilation log (including the NUL terminator). +/// @see nvrtcGetProgramLogSize +inline nvrtcResult +nvrtc_get_program_log_size(nvrtcProgram prog, size_t *logSizeRet) { + NV_FE_CALL_TO_NVRTC(nvrtc_get_program_log_size, nvrtcGetProgramLogSize, prog, logSizeRet); +} + +/// @brief Retrieve the compilation log. +/// @see nvrtcGetProgramLog +inline nvrtcResult +nvrtc_get_program_log(nvrtcProgram prog, char *log) { + NV_FE_CALL_TO_NVRTC(nvrtc_get_program_log, nvrtcGetProgramLog, prog, log); +} + +/// @brief Query the size of the compiled CUBIN image. +/// @see nvrtcGetCUBINSize +inline nvrtcResult +nvrtc_get_cubin_size(nvrtcProgram prog, size_t *cubinSizeRet) { + NV_FE_CALL_TO_NVRTC(nvrtc_get_cubin_size, nvrtcGetCUBINSize, prog, cubinSizeRet); +} + +/// @brief Retrieve the compiled CUBIN image. +/// @see nvrtcGetCUBIN +inline nvrtcResult +nvrtc_get_cubin(nvrtcProgram prog, char *cubin) { + NV_FE_CALL_TO_NVRTC(nvrtc_get_cubin, nvrtcGetCUBIN, prog, cubin); +} + +/// @brief Query the size of the generated PTX. +/// @see nvrtcGetPTXSize +inline nvrtcResult +nvrtc_get_ptx_size(nvrtcProgram prog, size_t *ptxSizeRet) { + NV_FE_CALL_TO_NVRTC(nvrtc_get_ptx_size, nvrtcGetPTXSize, prog, ptxSizeRet); +} + +/// @brief Retrieve the generated PTX. +/// @see nvrtcGetPTX +inline nvrtcResult +nvrtc_get_ptx(nvrtcProgram prog, char *ptx) { + NV_FE_CALL_TO_NVRTC(nvrtc_get_ptx, nvrtcGetPTX, prog, ptx); +} + +/// @brief Map an nvrtcResult code to a human-readable string. +/// @see nvrtcGetErrorString +inline const char * +nvrtc_get_error_string(nvrtcResult result) { +#if defined NV_CUDNN_FRONTEND_USE_DYNAMIC_LOADING + void *fptr = get_nvrtc_symbol("nvrtcGetErrorString"); + if (fptr == nullptr) { + return "NVRTC error (library not loaded)"; + } + return reinterpret_cast(fptr)(result); +#else + return nvrtcGetErrorString(result); +#endif +} + +} // namespace detail +} // namespace experimental +} // namespace cudnn_frontend diff --git a/deps/cudnn-frontend/include/cudnn_frontend/experimental/oss_engine_interface.h b/deps/cudnn-frontend/include/cudnn_frontend/experimental/oss_engine_interface.h new file mode 100644 index 0000000..7c0a38e --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/experimental/oss_engine_interface.h @@ -0,0 +1,261 @@ +/* + * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#pragma once + +#include "nvrtc_shim.h" +#include "attention_utils.h" +#include "../graph_helpers.h" +#include "../../cudnn_frontend_shim.h" + +#include + +#include +#include +#include +#include +#include +#include + +namespace cudnn_frontend { +namespace experimental { + +// ============================================================ +// IOssSdpaEngine — virtual interface for OSS SDPA engines +// Implemented by Sm90SdpaPrefillEngine and Sm100SdpaPrefillEngine. +// ============================================================ + +class IOssSdpaEngine { + public: + virtual ~IOssSdpaEngine() = default; + + virtual error_t + check_support(AttentionShape_t shape, int sm_version) = 0; + virtual error_t + build() = 0; + + virtual error_t + execute(int batch, + int heads_q, + int heads_kv, + int seq_q, + int seq_kv, + int d, + void* d_Q, + std::vector const& q_strides, + void* d_K, + std::vector const& k_strides, + void* d_V, + std::vector const& v_strides, + void* d_O, + std::vector const& o_strides, + void* d_max, + std::vector const& max_strides, + void* d_sum_exp, + std::vector const& se_strides, + void* workspace, + int device, + cudaStream_t stream, + std::optional user_attn_scale = std::nullopt) = 0; + + static int64_t + get_workspace_size() { + return 16; + } +}; + +// ============================================================ +// IOssNormEngine — virtual interface for OSS norm+activation engines. +// Implemented by Sm100RmsNormSiluEngine (SM100/Blackwell). +// Future implementations: Sm90, Sm110, LayerNorm variants, etc. +// ============================================================ + +// RmsNormSiluDtype is defined in sm100_rms_norm_silu_knobs.h (self-contained, +// also used by standalone tests without the framework). All arch-specific +// engine implementations share this enum. +// +// Forward-declared here; concrete implementations #include the knobs header. +enum class RmsNormSiluDtype : uint8_t; + +// Problem shape for RmsNorm+SiLU (extensible for future fields). +struct NormSiluShape_t { + int C = 0; // hidden dimension (columns) + int num_tokens = 0; // number of rows + RmsNormSiluDtype output_dtype{}; // output data type (bf16, fp8, nvfp4) +}; + +// Optional parameters for FP8 / NVFP4 output modes. +struct RmsNormSiluExtraParams { + // FP8 output: quantization scale (float*). If nullptr, uses 1.0 from workspace. + void* fp8_scale = nullptr; + void* fp8_scale_inv = nullptr; // optional: written by kernel if hasScaleInv + void* fp8_amax = nullptr; // optional: written by kernel if hasAmax + + // NVFP4 (1D1X1X) output: row-wise block-scale factors (float*, written by kernel) + // Shape: [num_tokens, ceil(C / 16)] + void* nvfp4_scale_row = nullptr; +}; + +class IOssNormEngine { + public: + virtual ~IOssNormEngine() = default; + + // Phase 1: Validate that (shape, sm_version) is supported. + virtual error_t + check_support(NormSiluShape_t shape, int sm_version) = 0; + + // Phase 2: NVRTC compile the kernel for the selected config. + virtual error_t + build() = 0; + + // Phase 3: Launch the kernel. + // input — [num_tokens, C] bf16 input tensor + // output — [num_tokens, C] output tensor (bf16, fp8, or nvfp4) + // weight — [C] bf16 gamma weights + // bias — [C] bf16 beta bias (can be nullptr) + // rows — num_tokens + // cols — C (hidden dimension) + // epsilon — RMSNorm epsilon (for L2Norm equiv: eps_l2 / C) + // extra — optional FP8/NVFP4 params (can be empty) + virtual error_t + execute(void* input, + void* output, + void* weight, + void* bias, + int rows, + int cols, + float epsilon, + void* workspace, + int device, + cudaStream_t stream, + RmsNormSiluExtraParams const& extra = {}) = 0; + + // Workspace size in bytes. The kernel needs an `rs` (inverse RMS) buffer, + // plus a float for FP8 default scale if no external scale is provided. + virtual int64_t + get_workspace_size() const = 0; +}; + +// ============================================================ +// Shared NVRTC compilation + module loading +// Used by both Sm90 and Sm100 engines to avoid code duplication. +// ============================================================ + +inline error_t +compile_and_load_kernel(const KernelSpec* spec, + cudaLibrary_t& module, + cudaKernel_t& kernelPtr, + std::unique_ptr& cubin, + size_t& cubinSize, + std::string& kernelName) { + // Ensure NVRTC library is available + RETURN_CUDNN_FRONTEND_ERROR_IF( + !detail::nvrtc_is_loaded(), error_code_t::CUDA_API_FAILED, "NVRTC library could not be loaded"); + + // Parse flags from embedded string + std::vector flags = parse_flags_string(spec->flags_raw, spec->flags_len); + std::vector flag_ptrs; + flag_ptrs.reserve(flags.size()); + for (auto& f : flags) { + flag_ptrs.push_back(f.c_str()); + } + + // Create NVRTC program from embedded source + nvrtcProgram prog; + nvrtcResult nvrtc_err; + + nvrtc_err = detail::nvrtc_create_program(&prog, spec->source, "kernel.cu", 0, nullptr, nullptr); + RETURN_CUDNN_FRONTEND_ERROR_IF( + nvrtc_err != NVRTC_SUCCESS, error_code_t::CUDA_API_FAILED, "nvrtcCreateProgram failed"); + + // Compile + nvrtcResult compResult = detail::nvrtc_compile_program(prog, (int)flag_ptrs.size(), flag_ptrs.data()); + + if (cudnn_frontend::isLoggingEnabled()) { + // Write the source to a file for debugging + std::ofstream ofs("kernel.cu"); + ofs << spec->source; + + // Try to dump PTX (cicc may have succeeded even if ptxas failed) + size_t ptxSize = 0; + if (detail::nvrtc_get_ptx_size(prog, &ptxSize) == NVRTC_SUCCESS && ptxSize > 1) { + std::vector ptx(ptxSize); + if (detail::nvrtc_get_ptx(prog, ptx.data()) == NVRTC_SUCCESS) { + std::ofstream ptx_ofs("kernel.ptx"); + ptx_ofs.write(ptx.data(), static_cast(ptxSize - 1)); + } + } + } + + if (compResult != NVRTC_SUCCESS) { + // Try to retrieve the compilation log for diagnostics + size_t logSize = 0; + detail::nvrtc_get_program_log_size(prog, &logSize); + std::string log_msg = "NVRTC compilation failed"; + if (logSize > 1) { + std::vector log(logSize); + detail::nvrtc_get_program_log(prog, log.data()); + log_msg += ": "; + log_msg += log.data(); + } + + CUDNN_FE_LOG_LABEL_ENDL("ERROR: NVRTC compilation failed: " << log_msg); + detail::nvrtc_destroy_program(&prog); + return {error_code_t::CUDA_API_FAILED, log_msg}; + } + + // Extract CUBIN + nvrtc_err = detail::nvrtc_get_cubin_size(prog, &cubinSize); + RETURN_CUDNN_FRONTEND_ERROR_IF( + nvrtc_err != NVRTC_SUCCESS, error_code_t::CUDA_API_FAILED, "nvrtcGetCUBINSize failed"); + + cubin = std::make_unique(cubinSize); + + nvrtc_err = detail::nvrtc_get_cubin(prog, cubin.get()); + RETURN_CUDNN_FRONTEND_ERROR_IF(nvrtc_err != NVRTC_SUCCESS, error_code_t::CUDA_API_FAILED, "nvrtcGetCUBIN failed"); + + nvrtc_err = detail::nvrtc_destroy_program(&prog); + RETURN_CUDNN_FRONTEND_ERROR_IF( + nvrtc_err != NVRTC_SUCCESS, error_code_t::CUDA_API_FAILED, "nvrtcDestroyProgram failed"); + + // Load module + extract kernel function + cudaError_t cuda_err; + + cuda_err = detail::cuda_library_load_data(&module, cubin.get(), nullptr, nullptr, 0, nullptr, nullptr, 0); + RETURN_CUDNN_FRONTEND_ERROR_IF(cuda_err != cudaSuccess, + error_code_t::CUDA_API_FAILED, + "cudaLibraryLoadData failed (cubin_size=" + std::to_string(cubinSize) + + ", error=" + detail::cuda_error_to_string(cuda_err) + ")"); + + kernelName = spec->kernel_name; + cuda_err = detail::cuda_library_get_kernel(&kernelPtr, module, kernelName.c_str()); + RETURN_CUDNN_FRONTEND_ERROR_IF( + cuda_err != cudaSuccess, + error_code_t::CUDA_API_FAILED, + "cudaLibraryGetKernel failed: " + detail::cuda_error_to_string(cuda_err) + " (kernel: " + kernelName + ")"); + + CUDNN_FE_LOG_LABEL_ENDL("INFO: NVRTC compilation successful, kernel: " << kernelName); + return {error_code_t::OK, ""}; +} + +} // namespace experimental +} // namespace cudnn_frontend diff --git a/deps/cudnn-frontend/include/cudnn_frontend/experimental/sm100_rms_norm_silu_engine.h b/deps/cudnn-frontend/include/cudnn_frontend/experimental/sm100_rms_norm_silu_engine.h new file mode 100644 index 0000000..bb7048c --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/experimental/sm100_rms_norm_silu_engine.h @@ -0,0 +1,697 @@ +/* + * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#pragma once + +#include "oss_engine_interface.h" +#include "sm100_rms_norm_silu_knobs.h" +// LN headers and kernel source for NVRTC compilation. +#include "../generated/rms_norm_silu/sm100/ln_headers.h" +#include "../generated/rms_norm_silu/sm100/ln_fwd_silu_kernel.h" + +#include +#include +#include +#include +#include + +namespace cudnn_frontend { +namespace experimental { + +// ============================================================ +// PersistentLnFwdParams — matches the kernel's expected parameter struct. +// Matches the kernel's expected parameter struct layout (PersistentLnFwdParams). +// ============================================================ + +struct PersistentLnFwdParams { + int ctas_per_col = 0; + int rows = 0; + int cols = 0; + int batchSize = 1; + int seqLen = 1; + + void* x = nullptr; + void* mu = nullptr; + void* rs = nullptr; + void* gamma = nullptr; + + void* workspace = nullptr; + int* barrier = nullptr; + + bool isRMSNorm = false; + bool noScale = false; + bool noBias = false; + bool isAdaLN = false; + bool isBatchFirst = true; + + // Output + void* z = nullptr; + void* beta = nullptr; + float epsilon = 0.f; + + // FP8 + bool fp8_out = false; + float* scale = nullptr; + float* scale_inv = nullptr; + float* amax = nullptr; + + // Block scale (NVFP4) — field order must match CuDNN's ln.h exactly + void* scale_row = nullptr; + void* scale_col = nullptr; // only used in 1d2x2x + void* z_col = nullptr; // only used in 1d2x2x + void* z_math = nullptr; // only used for 1d2x2x colwise kernel +}; + +// ============================================================ +// reduced_divisor — matches the kernel's expected second argument. +// Matches the kernel's expected second argument (reduced_divisor). +// ============================================================ + +struct reduced_divisor { + uint32_t mul_coeff = 0; + uint32_t shift_coeff = 0; + int32_t y = 0; + + reduced_divisor() = default; + reduced_divisor(int32_t _y) : y(_y) { + if (_y <= 1) { + mul_coeff = 0; + shift_coeff = 0; + return; + } + // Find shift such that 2^(32+shift) / y fits in 32 bits + for (shift_coeff = 0; shift_coeff < 32; ++shift_coeff) { + uint64_t one = 1; + uint64_t num = (one << (32 + shift_coeff)); + uint64_t den = static_cast(_y); + if (num / den < (one << 32)) { + mul_coeff = static_cast(num / den + 1); + break; + } + } + } +}; + +// ============================================================ +// Sm100RmsNormSiluEngine +// ============================================================ + +class Sm100RmsNormSiluEngine : public IOssNormEngine { + public: + Sm100RmsNormSiluEngine() = default; + + ~Sm100RmsNormSiluEngine() override { + if (module_) { + detail::cuda_library_unload(module_); + } + } + + // Non-copyable + Sm100RmsNormSiluEngine(const Sm100RmsNormSiluEngine&) = delete; + Sm100RmsNormSiluEngine& + operator=(const Sm100RmsNormSiluEngine&) = delete; + + // ---- Phase 1: Support check + knob selection ---- + error_t + check_support(NormSiluShape_t shape, int sm_version) override { + RETURN_CUDNN_FRONTEND_ERROR_IF( + sm_version < 80, error_code_t::GRAPH_NOT_SUPPORTED, "RmsNormSiluEngine requires SM80+"); + + // FP8 output requires SM89+ (Ada Lovelace / Hopper) + RETURN_CUDNN_FRONTEND_ERROR_IF(shape.output_dtype == RmsNormSiluDtype::FP8 && sm_version < 89, + error_code_t::GRAPH_NOT_SUPPORTED, + "FP8 output requires SM89+ (Ada/Hopper)"); + + // NVFP4 output requires SM100+ (Blackwell) + RETURN_CUDNN_FRONTEND_ERROR_IF(shape.output_dtype == RmsNormSiluDtype::NVFP4 && sm_version < 100, + error_code_t::GRAPH_NOT_SUPPORTED, + "NVFP4 output requires SM100+ (Blackwell)"); + + sm_version_ = sm_version; + + // Look up knobs: SM100 uses sweep-tuned LUT, other archs use fallback heuristic + knobs_ = lookup_rms_norm_silu_knobs(shape.C, shape.num_tokens, shape.output_dtype, sm_version); + RETURN_CUDNN_FRONTEND_ERROR_IF( + knobs_ == nullptr, + error_code_t::GRAPH_NOT_SUPPORTED, + "Unsupported problem size: C=" + std::to_string(shape.C) + " tokens=" + std::to_string(shape.num_tokens)); + + C_ = shape.C; + num_tokens_ = shape.num_tokens; + output_dtype_ = shape.output_dtype; + bytes_per_ldg_ = knobs_->bytes_per_ldg; + + // Derived config + warps_m_ = knobs_->warps_m; + warps_n_ = 1; // Always 1 + threads_per_cta_ = warps_m_ * warps_n_ * 32; + + // Estimate CTAS_PER_ROW when the sweep found column splitting optimal. + // Matches CuDNN's estimate_ctas_per_row() from LayerNorm_common.cpp: + // Find largest LDGS per CTA that avoids register spilling (< 64/NUM_ELTS), + // then CTAS_PER_ROW = total_ldgs / ldgs_per_cta. + // kernel_cfg=2 (non-persistent) requires CTAS_PER_ROW=1. + if (knobs_->split_cols == 4 && knobs_->kernel_cfg != 2) { + int input_size = 2; // bf16 + int num_elts = bytes_per_ldg_ / input_size; + int elts_per_ldg = num_elts * warps_n_ * 32; + if (elts_per_ldg > 0 && shape.C % elts_per_ldg == 0) { + int ldgs_per_row = shape.C / elts_per_ldg; + int ldgs_to_cause_register_spill = (num_elts > 0) ? (64 / num_elts) : 1; + ctas_per_row_ = 1; + for (int ldgs = std::min(ldgs_per_row, ldgs_to_cause_register_spill - 1); ldgs > 0; ldgs--) { + if (ldgs_per_row % ldgs == 0) { + ctas_per_row_ = ldgs_per_row / ldgs; + break; + } + } + } else { + ctas_per_row_ = 1; + } + } else { + ctas_per_row_ = 1; + } + is_cooperative_ = (ctas_per_row_ > 1); + + // Cache SM count + cudaDeviceProp devProp; + int device_ordinal = 0; + cudaError_t cuda_err = detail::cuda_get_device(&device_ordinal); + RETURN_CUDNN_FRONTEND_ERROR_IF(cuda_err != cudaSuccess, error_code_t::CUDA_API_FAILED, "cudaGetDevice failed"); + cuda_err = detail::cuda_get_device_properties(&devProp, device_ordinal); + RETURN_CUDNN_FRONTEND_ERROR_IF( + cuda_err != cudaSuccess, error_code_t::CUDA_API_FAILED, "cudaGetDeviceProperties failed"); + sm_count_ = devProp.multiProcessorCount; + + support_checked_ = true; + return {error_code_t::OK, ""}; + } + + // Workspace layout (all offsets 128-byte aligned): + // [0] rs : num_tokens * sizeof(float) + // [aligned] fp8_scale : sizeof(float) + // [aligned] scale_row : num_tokens * ceil(C/16) bytes (NVFP4 only) + // [aligned] coop_workspace : ctas_per_col * WARPS_M * CTAS_PER_ROW * sizeof(float2) * 2 + // [aligned] coop_barrier : 2 * ctas_per_col * sizeof(int32_t) + // + 128 padding + int64_t + get_workspace_size() const override { + int64_t ws = static_cast(num_tokens_) * sizeof(float); // rs + ws = ((ws + 127) / 128) * 128; + ws += sizeof(float); // FP8 default scale + ws = ((ws + 127) / 128) * 128; + if (output_dtype_ == RmsNormSiluDtype::NVFP4) { + ws += static_cast(num_tokens_) * ((C_ + 15) / 16); + ws = ((ws + 127) / 128) * 128; + } + if (is_cooperative_) { + int ctas_per_col = std::max(1, (num_tokens_ + warps_m_ - 1) / warps_m_); + // Inter-CTA partial stats: ctas_per_col * WARPS_M * CTAS_PER_ROW * sizeof(float2) * 2 + ws += static_cast(ctas_per_col) * warps_m_ * ctas_per_row_ * sizeof(float) * 2 * 2; + ws = ((ws + 127) / 128) * 128; + // Barrier array: 2 * ctas_per_col * sizeof(int32_t) + ws += 2 * static_cast(ctas_per_col) * sizeof(int32_t); + ws = ((ws + 127) / 128) * 128; + } + ws += 128; // final alignment + return ws; + } + + // Get offset of scale_row within workspace (for reading back in tests). + // Returns -1 if not NVFP4 output. + int64_t + get_scale_row_workspace_offset() const { + if (output_dtype_ != RmsNormSiluDtype::NVFP4) return -1; + int64_t off = static_cast(num_tokens_) * sizeof(float); + off = ((off + 127) / 128) * 128; + off += sizeof(float); + off = ((off + 127) / 128) * 128; + return off; + } + + int64_t + get_scale_row_size_bytes() const { + if (output_dtype_ != RmsNormSiluDtype::NVFP4) return 0; + return static_cast(num_tokens_) * ((C_ + 15) / 16); + } + + // ---- Phase 2: NVRTC compilation ---- + error_t + build() override { + RETURN_CUDNN_FRONTEND_ERROR_IF(!support_checked_ || knobs_ == nullptr, + error_code_t::INVALID_VALUE, + "build() called before check_support()"); + + // Assemble the full kernel source: + // 1. Generate constexpr defines from knobs + // 2. Prepend to headers + kernel body + + // Order: preamble → headers (defines cuda types) → constexpr defines + // (uses those types) → kernel body (uses everything). + std::string defines = generate_constexpr_defines(); + + // Undefine EWP/RDC to avoid removed cudaCGScopeMultiGrid in cooperative_groups + std::string preamble = "#undef __CUDACC_EWP__\n#undef __CUDACC_RDC__\n"; + + std::string full_source = preamble + std::string(generated::ln_headers_source) + defines + + std::string(generated::ln_fwd_silu_kernel_source); + + // Parse compile flags and override --gpu-architecture for the target SM version + std::vector flags = + parse_flags_string(generated::ln_fwd_silu_kernel_flags, generated::ln_fwd_silu_kernel_flags_len); + + // Replace the embedded --gpu-architecture with the correct arch for this GPU. + // The 'a' suffix = architecture-specific (enables SM-specific features like redux.sync). + // The 'f' suffix = forward-compatible across an SM family (e.g., sm_100f works on SM100-SM103). + // The kernel source has #if __CUDA_ARCH__ guards for SM-specific instructions. + { + std::string target_arch; + if (sm_version_ == 100) + target_arch = "sm_100a"; // B200: arch-specific for best codegen + else if (sm_version_ > 100 && sm_version_ < 110) + target_arch = "sm_100f"; // GB300 etc: forward-compat + else if (sm_version_ == 90) + target_arch = "sm_90a"; + else if (sm_version_ > 90 && sm_version_ < 100) + target_arch = "sm_90"; // SM92 etc + else if (sm_version_ >= 89) + target_arch = "sm_89"; + else if (sm_version_ >= 86) + target_arch = "sm_86"; + else + target_arch = "sm_80"; + + for (auto& f : flags) { + if (f.find("--gpu-architecture=") == 0) { + f = "--gpu-architecture=" + target_arch; + break; + } + } + } + + // Add CUDA Toolkit include paths for NVRTC to resolve #include etc. + std::string cuda_include = "/usr/local/cuda/include"; + if (auto env0 = get_environment("CUDA_HOME")) { + cuda_include = std::string(env0) + "/include"; + } else if (auto env1 = get_environment("CUDA_PATH")) { + cuda_include = std::string(env1) + "/include"; + } + flags.push_back("--include-path=" + cuda_include); + + // CCCL headers needed for cuda::ptx:: used by block-scale (NVFP4) code paths. + // CUDA 13+ ships CCCL under targets/-linux/include/cccl. + // Add both x86_64 and aarch64 paths — NVRTC silently ignores non-existent paths. + std::string cuda_base = cuda_include.substr(0, cuda_include.rfind("/include")); + for (const char* arch : {"x86_64", "aarch64"}) { + std::string target_dir = cuda_base + "/targets/" + arch + "-linux/include"; + flags.push_back("--include-path=" + target_dir); + flags.push_back("--include-path=" + target_dir + "/cccl"); + } + + std::vector flag_ptrs; + for (auto& f : flags) flag_ptrs.push_back(f.c_str()); + + // NVRTC compile + RETURN_CUDNN_FRONTEND_ERROR_IF( + !detail::nvrtc_is_loaded(), error_code_t::CUDA_API_FAILED, "NVRTC library could not be loaded"); + + nvrtcProgram prog; + nvrtcResult nvrtc_err; + + nvrtc_err = detail::nvrtc_create_program(&prog, full_source.c_str(), "ln_fwd_silu.cu", 0, nullptr, nullptr); + RETURN_CUDNN_FRONTEND_ERROR_IF( + nvrtc_err != NVRTC_SUCCESS, error_code_t::CUDA_API_FAILED, "nvrtcCreateProgram failed"); + + nvrtcResult compResult = detail::nvrtc_compile_program(prog, (int)flag_ptrs.size(), flag_ptrs.data()); + if (compResult != NVRTC_SUCCESS) { + size_t logSize = 0; + detail::nvrtc_get_program_log_size(prog, &logSize); + std::string log_msg = "NVRTC compilation failed"; + if (logSize > 1) { + std::vector log(logSize); + detail::nvrtc_get_program_log(prog, log.data()); + log_msg += ": "; + log_msg += log.data(); + } + detail::nvrtc_destroy_program(&prog); + return {error_code_t::CUDA_API_FAILED, log_msg}; + } + + // Extract CUBIN + nvrtc_err = detail::nvrtc_get_cubin_size(prog, &cubinSize_); + RETURN_CUDNN_FRONTEND_ERROR_IF( + nvrtc_err != NVRTC_SUCCESS, error_code_t::CUDA_API_FAILED, "nvrtcGetCUBINSize failed"); + + cubin_ = std::make_unique(cubinSize_); + nvrtc_err = detail::nvrtc_get_cubin(prog, cubin_.get()); + RETURN_CUDNN_FRONTEND_ERROR_IF( + nvrtc_err != NVRTC_SUCCESS, error_code_t::CUDA_API_FAILED, "nvrtcGetCUBIN failed"); + + detail::nvrtc_destroy_program(&prog); + + // Load module + kernel + cudaError_t cuda_err; + cuda_err = detail::cuda_library_load_data(&module_, cubin_.get(), nullptr, nullptr, 0, nullptr, nullptr, 0); + RETURN_CUDNN_FRONTEND_ERROR_IF(cuda_err != cudaSuccess, + error_code_t::CUDA_API_FAILED, + "cudaLibraryLoadData failed: " + detail::cuda_error_to_string(cuda_err)); + + cuda_err = detail::cuda_library_get_kernel(&kernelPtr_, module_, "ln_fwd_kernel"); + RETURN_CUDNN_FRONTEND_ERROR_IF(cuda_err != cudaSuccess, + error_code_t::CUDA_API_FAILED, + "cudaLibraryGetKernel failed: " + detail::cuda_error_to_string(cuda_err)); + + built_ = true; + return {error_code_t::OK, ""}; + } + + // ---- Phase 3: Execute ---- + error_t + execute(void* input, + void* output, + void* weight, // gamma + void* bias, // beta (can be nullptr) + int rows, + int cols, + float epsilon, + void* workspace, + int device, + cudaStream_t stream, + RmsNormSiluExtraParams const& extra = {}) override { + RETURN_CUDNN_FRONTEND_ERROR_IF(!built_, error_code_t::INVALID_VALUE, "execute() called before build()"); + + // Compute grid dimensions + int ctas_per_col_max = (rows + warps_m_ - 1) / warps_m_; + int ctas_per_col; + if (knobs_->kernel_cfg == 2) { + // Non-persistent mode: launch one CTA group per row (max CTAs). + // CuDNN overrides ctas_per_col to max when use_non_persistent_mode=true. + ctas_per_col = ctas_per_col_max; + } else { + ctas_per_col = std::min(sm_count_ * static_cast(knobs_->occupancy) / ctas_per_row_, ctas_per_col_max); + } + ctas_per_col = std::max(ctas_per_col, 1); + + dim3 grid(ctas_per_row_ * ctas_per_col); + dim3 block(threads_per_cta_); + + // Pack params + PersistentLnFwdParams params{}; + params.rows = rows; + params.cols = cols; + params.ctas_per_col = ctas_per_col; + params.isRMSNorm = true; + params.noScale = (weight == nullptr); + params.noBias = (bias == nullptr); + params.isBatchFirst = true; + params.batchSize = 1; + params.seqLen = rows; + params.epsilon = epsilon; + params.x = input; + params.z = output; + params.gamma = weight; + params.beta = bias; + + // Workspace layout (128-byte aligned): + // [0] rs : rows * sizeof(float) + // [aligned] fp8_scale : sizeof(float) + // [aligned] scale_row : rows * ceil(cols/16) bytes (NVFP4 only) + char* ws_ptr = static_cast(workspace); + params.rs = ws_ptr; + { + int64_t off = static_cast(rows) * sizeof(float); + off = ((off + 127) / 128) * 128; + ws_ptr = static_cast(workspace) + off; + } + + // FP8 output: set scale pointer + if (output_dtype_ == RmsNormSiluDtype::FP8) { + params.fp8_out = true; + if (extra.fp8_scale) { + params.scale = static_cast(extra.fp8_scale); + } else { + // Use default scale = 1.0 from workspace (device memory) + // Write 1.0f (IEEE 754: 0x3F800000) asynchronously on stream + float* default_scale = reinterpret_cast(ws_ptr); + cudaError_t memset_err = detail::cuda_mem_set_d32_async(default_scale, 0x3F800000u, 1, stream); + RETURN_CUDNN_FRONTEND_ERROR_IF(memset_err != cudaSuccess, + error_code_t::CUDA_API_FAILED, + "cudaMemcpyAsync for default scale failed"); + params.scale = default_scale; + } + params.scale_inv = static_cast(extra.fp8_scale_inv); // may be nullptr + params.amax = static_cast(extra.fp8_amax); // may be nullptr + } + + // NVFP4 output: set scale_row pointer + if (output_dtype_ == RmsNormSiluDtype::NVFP4) { + if (extra.nvfp4_scale_row) { + params.scale_row = extra.nvfp4_scale_row; + } else { + // Auto-allocate from workspace after fp8_scale slot + int64_t scale_row_off = get_scale_row_workspace_offset(); + params.scale_row = static_cast(workspace) + scale_row_off; + } + } + + // Multi-CTA cooperative reduction: allocate workspace + barrier from workspace + if (is_cooperative_) { + // Compute workspace offset after rs + fp8_scale + scale_row + int64_t coop_off = static_cast(rows) * sizeof(float); + coop_off = ((coop_off + 127) / 128) * 128; + coop_off += sizeof(float); + coop_off = ((coop_off + 127) / 128) * 128; + if (output_dtype_ == RmsNormSiluDtype::NVFP4) { + coop_off += static_cast(rows) * ((cols + 15) / 16); + coop_off = ((coop_off + 127) / 128) * 128; + } + + // Inter-CTA partial stats workspace + int64_t coop_ws_size = + static_cast(ctas_per_col) * warps_m_ * ctas_per_row_ * sizeof(float) * 2 * 2; + params.workspace = static_cast(workspace) + coop_off; + coop_off += coop_ws_size; + coop_off = ((coop_off + 127) / 128) * 128; + + // Barrier array — must be zeroed before each launch + int64_t barrier_count = 2 * ctas_per_col; + params.barrier = reinterpret_cast(static_cast(workspace) + coop_off); + cudaError_t memset_err = + detail::cuda_mem_set_async(params.barrier, 0, barrier_count * sizeof(int32_t), stream); + RETURN_CUDNN_FRONTEND_ERROR_IF( + memset_err != cudaSuccess, + error_code_t::CUDA_API_FAILED, + "cudaMemsetAsync failed for cooperative barrier (count=" + std::to_string(barrier_count) + + " offset=" + std::to_string(coop_off) + ")"); + } + + // reduced_divisor for batch/seqLen (batchFirst=true, so divide by seqLen=rows) + reduced_divisor divisor(rows); + + // Kernel args + void* kernelParams[] = { + reinterpret_cast(¶ms), + reinterpret_cast(&divisor), + }; + + // Static shared memory: USE_STATIC_SMEM_VALUE is defined in the NVRTC source, + // so the kernel allocates smem at compile time. Pass 0 for dynamic smem. + // The kernel uses compile-time static shared memory (no dynamic smem needed). + int smem_bytes = 0; + + // Set shared memory attribute + cudaError_t cuda_err; + cuda_err = detail::cuda_kernel_set_attribute_for_device( + kernelPtr_, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_bytes, device); + RETURN_CUDNN_FRONTEND_ERROR_IF( + cuda_err != cudaSuccess, + error_code_t::CUDA_API_FAILED, + "cudaKernelSetAttributeForDevice failed: " + detail::cuda_error_to_string(cuda_err)); + + // Launch + cuda_err = detail::cuda_launch_kernel((const void*)kernelPtr_, grid, block, kernelParams, smem_bytes, stream); + RETURN_CUDNN_FRONTEND_ERROR_IF(cuda_err != cudaSuccess, + error_code_t::CUDA_API_FAILED, + "cudaLaunchKernel failed: " + detail::cuda_error_to_string(cuda_err) + + " (grid=" + std::to_string(grid.x) + " block=" + std::to_string(block.x) + + " smem=" + std::to_string(smem_bytes) + ")"); + + return {error_code_t::OK, ""}; + } + + private: + std::string + generate_constexpr_defines() const { + std::string s; + + // Type aliases (bf16 input always for VAE) + s += "\nusing ITYPE = nv_bfloat16;"; + switch (output_dtype_) { + case RmsNormSiluDtype::BF16: + s += "\nusing OTYPE = nv_bfloat16;"; + break; + case RmsNormSiluDtype::FP8: + s += "\nusing OTYPE = nv_fp8_e4m3;"; + break; + case RmsNormSiluDtype::NVFP4: + s += "\nusing OTYPE = nv_fp4_e2m1;"; + break; + } + s += "\nusing WTYPE = nv_bfloat16;"; + s += "\nusing CTYPE = float;"; + // NORM_OTYPE must match the norm output type, not compute type. + // Must match the norm output type, not compute type. + // For bf16 output: nv_bfloat16. For FP8/NVFP4: float (block-scale intermediate). + switch (output_dtype_) { + case RmsNormSiluDtype::BF16: + s += "\nusing NORM_OTYPE = nv_bfloat16;"; + break; + case RmsNormSiluDtype::FP8: + case RmsNormSiluDtype::NVFP4: + s += "\nusing NORM_OTYPE = float;"; + break; + } + + // Kernel config + s += "\nconstexpr int HIDDEN_SIZE = " + std::to_string(C_) + ";"; + s += "\nconstexpr int BATCH_SIZE = 1;"; + s += "\nconstexpr int CTAS_PER_ROW = " + std::to_string(ctas_per_row_) + ";"; + s += "\nconstexpr int WARPS_M = " + std::to_string(warps_m_) + ";"; + s += "\nconstexpr int WARPS_N = " + std::to_string(warps_n_) + ";"; + s += "\nconstexpr int BYTES_PER_LDG = " + std::to_string(bytes_per_ldg_) + ";"; + s += "\nconstexpr bool isRMSNorm = 1;"; + s += "\nconstexpr bool isAdaLN = 0;"; + s += "\nconstexpr bool isBatchFirst = 1;"; + s += "\nconstexpr bool hasGamma = 1;"; + s += "\nconstexpr bool hasBeta = 0;"; + s += "\nconstexpr bool isZeroCenteredGamma = 0;"; + s += "\nconstexpr bool isZeroCenteredGammaCastBeforeAdd = 0;"; + + bool use_smem_gamma = (knobs_->kernel_cfg == 1); + bool use_non_persistent = (knobs_->kernel_cfg == 2); + s += "\nconstexpr bool useSmemGamma = " + std::to_string(use_smem_gamma ? 1 : 0) + ";"; + s += "\nconstexpr bool GAMMA_ON_DEMAND = " + std::to_string((!use_smem_gamma && use_non_persistent) ? 1 : 0) + + ";"; + + bool is_fp8 = (output_dtype_ == RmsNormSiluDtype::FP8); + bool is_nvfp4 = (output_dtype_ == RmsNormSiluDtype::NVFP4); + s += "\nconstexpr bool isFP8Out = " + std::to_string(is_fp8 ? 1 : 0) + ";"; + s += "\nconstexpr bool hasScaleInv = 0;"; + s += "\nconstexpr bool hasAmax = 0;"; + s += "\n#define LN_USE_CLUSTER 0"; + s += "\nconstexpr bool USE_CLUSTER = 0;"; + s += "\nconstexpr bool isBlockScaleOut = " + std::to_string(is_nvfp4 ? 1 : 0) + ";"; + s += "\nconstexpr bool isFP4Out = " + std::to_string(is_nvfp4 ? 1 : 0) + ";"; + s += "\nconstexpr bool isBlockScale_1D1X1X = " + std::to_string(is_nvfp4 ? 1 : 0) + ";"; + s += "\nconstexpr bool isBlockScale_1D2X2X = 0;"; + s += "\nconstexpr bool isBlockScale_1D2X2X_Transpose = 0;"; + s += "\nconstexpr bool useBlockScaleColwiseKernel = 0;"; + s += "\nconstexpr int DESIRED_OCCUPANCY = " + std::to_string(knobs_->occupancy) + ";"; + + // Ktraits instantiation + s += "\nusing Ktraits = Kernel_traits: + // __align__(16) char smem_stats[SMEM_STATS_ELEMENTS] — 1 or Stats::SMEM_BYTES + // __align__(16) uint64_t smem_bar[SMEM_BAR_ELEMENTS] — 1 (no cluster) + // __align__(16) weight_t smem_gamma[GAMMA_ELEMENTS] — 1 or LDGS*THREADS_PER_ROW*NUM_ELTS + // __align__(16) weight_t smem_beta[BETA_ELEMENTS] — 1 (no beta) + // __align__(16) float smem_mxfp8[SMEM_MXFP8_ELEMENTS] — 1 (no 1D2X2X) + int + compute_smem_bytes() const { + int elts_per_ldg = bytes_per_ldg_ / 2; + int vec_cols = C_ / elts_per_ldg; + int threads_per_row = warps_n_ * 32; + int vec_cols_per_ldg = ctas_per_row_ * threads_per_row; + int ldgs = (vec_cols_per_ldg > 0) ? (vec_cols / vec_cols_per_ldg) : 1; + int num_elts = elts_per_ldg; + + auto align16 = [](int bytes) { return ((bytes + 15) / 16) * 16; }; + + // smem_stats: min 1 byte → 16 when aligned. + // When CTAS_PER_ROW=1 && WARPS_N=1, Stats::SMEM_BYTES=0, so element count = max(0,1) = 1. + int stats_bytes = 1; + if (ctas_per_row_ > 1 || warps_n_ > 1) { + stats_bytes = warps_m_ * warps_n_ * sizeof(float) * 2 * 2; + } + int smem_stats = align16(stats_bytes); + + // smem_bar: 1 uint64_t (no cluster) + int smem_bar = align16(static_cast(sizeof(uint64_t))); + + // smem_gamma: kernel_cfg=1 loads gamma to smem + int gamma_elements = 1; + if (knobs_->kernel_cfg == 1) { + gamma_elements = 1 * ldgs * threads_per_row * num_elts; // BATCH_SIZE=1 + } + int smem_gamma = align16(gamma_elements * 2); // weight_t = bf16 = 2 bytes + + // smem_beta: 1 element (hasBeta=false) + int smem_beta = align16(2); // 1 * sizeof(bf16) + + // smem_mxfp8: 1 element (not 1D2X2X) + int smem_mxfp8 = align16(static_cast(sizeof(float))); + + int smem_bytes = smem_stats + smem_bar + smem_gamma + smem_beta + smem_mxfp8; + smem_bytes = ((smem_bytes + 127) / 128) * 128; + return smem_bytes; + } + + // State from check_support() + const RmsNormSiluKnobs* knobs_ = nullptr; + int C_ = 0; + int num_tokens_ = 0; + RmsNormSiluDtype output_dtype_ = RmsNormSiluDtype::BF16; + int bytes_per_ldg_ = 0; + int warps_m_ = 0; + int warps_n_ = 0; + int threads_per_cta_ = 0; + int ctas_per_row_ = 0; + int sm_count_ = 0; + int sm_version_ = 0; + bool is_cooperative_ = false; + bool support_checked_ = false; + + // State from build() + cudaLibrary_t module_ = nullptr; + cudaKernel_t kernelPtr_ = nullptr; + std::unique_ptr cubin_; + size_t cubinSize_ = 0; + bool built_ = false; +}; + +} // namespace experimental +} // namespace cudnn_frontend diff --git a/deps/cudnn-frontend/include/cudnn_frontend/experimental/sm100_rms_norm_silu_knobs.h b/deps/cudnn-frontend/include/cudnn_frontend/experimental/sm100_rms_norm_silu_knobs.h new file mode 100644 index 0000000..8080ab2 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/experimental/sm100_rms_norm_silu_knobs.h @@ -0,0 +1,215 @@ +#pragma once + +// Auto-generated knob selection logic for Sm100RmsNormSiluEngine. +// Generated from optimal knob sweep results on B200 (SM100). +// Knob mapping: knobTileRows {0:1,1:4,2:8,3:32}, knobLoadSize {0:2,1:4,2:8,3:16} + +#include + +namespace cudnn_frontend { +namespace experimental { + +enum class RmsNormSiluDtype : uint8_t { + BF16 = 0, + FP8 = 1, + NVFP4 = 2, +}; + +// Compact knob configuration per (C, tokens, dtype) entry. +// WARPS_N is always 1. All values are the ACTUAL kernel parameters +// (not knob indices). +struct RmsNormSiluKnobs { + uint8_t warps_m; // WARPS_M value: 1, 4, 8, or 32 + uint8_t split_cols; // knobSplitCols: 0 = no split, 4 = estimated CTAS_PER_ROW + uint8_t kernel_cfg; // knobKernelCfg: 0, 1, or 2 + uint8_t occupancy; // DESIRED_OCCUPANCY: 0-16 + uint8_t bytes_per_ldg; // BYTES_PER_LDG: 2, 4, 8, or 16 +}; + +static constexpr int kSupportedC[] = {64, 128, 160, 256, 320, 512, 640, 1024}; +static constexpr int kSupportedTokens[] = {1560, 6240, 24960, 99840, 399360}; +static constexpr int kNumC = 8; +static constexpr int kNumTokens = 5; +static constexpr int kNumDtypes = 3; + +// Knob LUT indexed as: knob_lut[c_idx][tokens_idx][dtype_idx] +// c_idx: 0=64, 1=128, 2=160, 3=256, 4=320, 5=512, 6=640, 7=1024 +// tokens_idx: 0=1560, 1=6240, 2=24960, 3=99840, 4=399360 +// dtype_idx: 0=bf16, 1=fp8, 2=nvfp4 +static constexpr RmsNormSiluKnobs knob_lut[kNumC][kNumTokens][kNumDtypes] = { + { + // C=64 + {{8, 0, 0, 2, 4}, {8, 4, 0, 6, 4}, {8, 0, 2, 1, 4}}, // tokens=1560 + {{32, 4, 0, 2, 4}, {8, 0, 0, 3, 2}, {8, 4, 0, 4, 4}}, // tokens=6240 + {{32, 4, 0, 2, 4}, {8, 0, 0, 7, 4}, {8, 0, 1, 6, 4}}, // tokens=24960 + {{8, 0, 1, 8, 4}, {8, 0, 1, 6, 2}, {32, 4, 1, 2, 4}}, // tokens=99840 + {{4, 0, 1, 16, 4}, {32, 0, 1, 2, 2}, {32, 4, 1, 2, 4}}, // tokens=399360 + }, + { + // C=128 + {{8, 4, 0, 3, 4}, {8, 0, 0, 3, 4}, {8, 4, 0, 3, 8}}, // tokens=1560 + {{8, 0, 0, 3, 4}, {8, 0, 0, 4, 8}, {8, 0, 0, 5, 8}}, // tokens=6240 + {{8, 0, 0, 6, 4}, {8, 0, 0, 8, 8}, {8, 0, 1, 8, 8}}, // tokens=24960 + {{32, 4, 0, 2, 4}, {32, 0, 0, 2, 8}, {32, 0, 1, 2, 8}}, // tokens=99840 + {{8, 0, 0, 8, 4}, {32, 0, 0, 2, 8}, {32, 0, 1, 2, 8}}, // tokens=399360 + }, + { + // C=160 + {{8, 0, 0, 4, 2}, {8, 0, 0, 2, 2}, {4, 4, 0, 4, 2}}, // tokens=1560 + {{8, 0, 0, 4, 2}, {8, 0, 0, 4, 2}, {8, 0, 1, 4, 2}}, // tokens=6240 + {{8, 4, 1, 6, 2}, {8, 4, 0, 6, 2}, {8, 4, 1, 8, 2}}, // tokens=24960 + {{32, 4, 1, 2, 2}, {32, 4, 1, 2, 2}, {32, 4, 0, 1, 2}}, // tokens=99840 + {{32, 4, 1, 2, 2}, {32, 4, 1, 2, 2}, {32, 0, 1, 2, 2}}, // tokens=399360 + }, + { + // C=256 + {{8, 0, 0, 6, 16}, {8, 4, 0, 2, 4}, {8, 0, 2, 1, 16}}, // tokens=1560 + {{8, 0, 0, 4, 4}, {8, 0, 0, 4, 4}, {8, 0, 2, 1, 16}}, // tokens=6240 + {{8, 0, 0, 8, 16}, {8, 4, 0, 8, 16}, {8, 4, 1, 6, 16}}, // tokens=24960 + {{4, 4, 0, 16, 16}, {4, 0, 0, 16, 16}, {32, 0, 1, 1, 16}}, // tokens=99840 + {{4, 0, 0, 16, 16}, {32, 0, 0, 2, 16}, {32, 0, 1, 2, 16}}, // tokens=399360 + }, + { + // C=320 + {{8, 4, 1, 4, 4}, {8, 0, 0, 2, 4}, {4, 4, 0, 9, 4}}, // tokens=1560 + {{8, 4, 0, 5, 4}, {8, 0, 0, 5, 4}, {4, 0, 0, 9, 4}}, // tokens=6240 + {{8, 0, 0, 5, 4}, {8, 0, 0, 5, 4}, {8, 0, 1, 8, 4}}, // tokens=24960 + {{4, 0, 1, 16, 4}, {32, 0, 1, 2, 4}, {32, 4, 1, 2, 4}}, // tokens=99840 + {{32, 4, 0, 2, 4}, {32, 0, 1, 2, 4}, {32, 4, 1, 2, 4}}, // tokens=399360 + }, + { + // C=512 + {{8, 0, 0, 2, 16}, {8, 0, 0, 2, 8}, {4, 4, 0, 3, 16}}, // tokens=1560 + {{8, 0, 0, 5, 16}, {8, 0, 0, 4, 8}, {4, 0, 0, 9, 16}}, // tokens=6240 + {{4, 0, 0, 8, 16}, {4, 0, 0, 9, 8}, {4, 0, 2, 1, 16}}, // tokens=24960 + {{4, 0, 2, 1, 8}, {32, 4, 1, 2, 8}, {32, 4, 0, 1, 16}}, // tokens=99840 + {{4, 0, 2, 1, 4}, {32, 4, 1, 2, 8}, {32, 0, 0, 1, 16}}, // tokens=399360 + }, + { + // C=640 + {{4, 0, 0, 4, 4}, {4, 0, 0, 3, 8}, {4, 4, 0, 5, 8}}, // tokens=1560 + {{4, 0, 0, 5, 4}, {8, 0, 0, 4, 8}, {4, 0, 1, 9, 8}}, // tokens=6240 + {{4, 0, 0, 5, 4}, {8, 0, 0, 4, 8}, {4, 0, 2, 1, 8}}, // tokens=24960 + {{4, 0, 2, 1, 8}, {4, 4, 0, 9, 8}, {32, 0, 1, 1, 8}}, // tokens=99840 + {{4, 0, 2, 1, 8}, {32, 4, 1, 2, 8}, {32, 4, 1, 1, 8}}, // tokens=399360 + }, + { + // C=1024 + {{4, 4, 0, 3, 16}, {4, 0, 0, 3, 4}, {4, 4, 0, 7, 16}}, // tokens=1560 + {{4, 0, 0, 5, 16}, {4, 0, 0, 5, 8}, {4, 0, 2, 1, 16}}, // tokens=6240 + {{4, 4, 1, 10, 16}, {1, 4, 0, 16, 8}, {4, 0, 2, 1, 16}}, // tokens=24960 + {{8, 0, 2, 1, 16}, {4, 0, 1, 9, 8}, {32, 0, 1, 1, 16}}, // tokens=99840 + {{8, 0, 2, 1, 16}, {32, 4, 1, 1, 8}, {32, 4, 1, 1, 16}}, // tokens=399360 + }, +}; + +// Compute conservative default knobs for arbitrary problem sizes not in the LUT. +// Uses safe defaults (WARPS_M=1, BPL=4, occupancy=1) and validates vectorization +// divisibility constraints before accepting a configuration. +// Returns true if a valid configuration was found, false otherwise. +inline bool +compute_default_knobs(int C, int num_tokens, RmsNormSiluDtype dtype, RmsNormSiluKnobs& out) { + // Conservative defaults: + // CTAS_PER_ROW = 1, WARPS_M = 1, WARPS_N = 1, BPL = 4, occupancy = 1, kernel_cfg = 0 + // For block-scale output (NVFP4): WARPS_M = 32 + + int input_size = 2; // bf16 input always + + // Start with conservative defaults + int warps_m = (dtype == RmsNormSiluDtype::NVFP4) ? 32 : 1; + int warps_n = 1; // always 1 for our engine + int bpl = 4; // default bytes per load + int cpr = 1; // no column splitting for fallback + int occ = 1; + int kcfg = 0; + + // Validation: C must be evenly divisible into vectorized loads. + // NUM_ELTS = BYTES_PER_LDG / sizeof(input_t) + // VEC_COLS = C / NUM_ELTS + // VEC_COLS_PER_LDG = CTAS_PER_ROW * WARPS_N * 32 + // Require: C % NUM_ELTS == 0 AND VEC_COLS % VEC_COLS_PER_LDG == 0 + // Also: LDGS = VEC_COLS / VEC_COLS_PER_LDG <= 1024 (avoid register spill) + + auto validate = [&](int test_bpl, int test_wm) -> bool { + int num_elts = test_bpl / input_size; + if (num_elts <= 0 || C % num_elts != 0) return false; + int vec_cols = C / num_elts; + int vec_cols_per_ldg = cpr * warps_n * 32; + if (vec_cols_per_ldg <= 0 || vec_cols % vec_cols_per_ldg != 0) return false; + int ldgs = vec_cols / vec_cols_per_ldg; + if (ldgs > 1024) return false; // reject extreme LDGS to avoid register spilling + // Check WARPS_M constraint: if WARPS_M > 1, rows per CTA must divide evenly + if (test_wm > 1 && num_tokens % test_wm != 0) return false; + return true; + }; + + // Try default BPL=4, then cascade through {4, 8, 16, 2} + static constexpr int bpl_candidates[] = {4, 8, 16, 2}; + bool found = false; + for (int candidate : bpl_candidates) { + if (validate(candidate, warps_m)) { + bpl = candidate; + found = true; + break; + } + } + + // If WARPS_M=1 failed, try bumping to WARPS_M=4 for better row coverage + if (!found && warps_m == 1 && num_tokens % 4 == 0) { + warps_m = 4; + for (int candidate : bpl_candidates) { + if (validate(candidate, warps_m)) { + bpl = candidate; + found = true; + break; + } + } + } + + if (!found) return false; + + out.warps_m = static_cast(warps_m); + out.split_cols = 0; // no column splitting + out.kernel_cfg = static_cast(kcfg); + out.occupancy = static_cast(occ); + out.bytes_per_ldg = static_cast(bpl); + return true; +} + +// Look up knob configuration for a given (C, num_tokens, output_dtype, sm_version). +// Tier 1: exact LUT match for SM100 VAE problem sizes (optimal, sweep-tuned on B200). +// Tier 2: fallback heuristic for other archs or arbitrary sizes (functional, conservative). +// Returns nullptr only if the problem is fundamentally unsupported. +inline const RmsNormSiluKnobs* +lookup_rms_norm_silu_knobs(int C, int num_tokens, RmsNormSiluDtype dtype, int sm_version = 100) { + // Tier 1: exact LUT match — only valid for SM100 (swept on B200) + if (sm_version >= 100) { + int c_idx = -1, t_idx = -1; + for (int i = 0; i < kNumC; ++i) { + if (kSupportedC[i] == C) { + c_idx = i; + break; + } + } + for (int i = 0; i < kNumTokens; ++i) { + if (kSupportedTokens[i] == num_tokens) { + t_idx = i; + break; + } + } + if (c_idx >= 0 && t_idx >= 0) { + return &knob_lut[c_idx][t_idx][static_cast(dtype)]; + } + } + + // Tier 2: fallback heuristic for non-SM100 archs or non-LUT problem sizes + static thread_local RmsNormSiluKnobs fallback; + if (compute_default_knobs(C, num_tokens, dtype, fallback)) { + return &fallback; + } + + return nullptr; // fundamentally unsupported (C not divisible by any valid config) +} + +} // namespace experimental +} // namespace cudnn_frontend diff --git a/deps/cudnn-frontend/include/cudnn_frontend/experimental/sm100_sdpa_prefill_engine.h b/deps/cudnn-frontend/include/cudnn_frontend/experimental/sm100_sdpa_prefill_engine.h new file mode 100644 index 0000000..73da07d --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/experimental/sm100_sdpa_prefill_engine.h @@ -0,0 +1,544 @@ +/* + * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#pragma once + +#include "oss_engine_interface.h" +#include "../generated/sdpa/sm100/prefill/full_seqlens/d128_fprop_kernel.h" +#include "../generated/sdpa/sm100/prefill/full_seqlens/d64_fprop_kernel.h" + +#include + +// Suppress MSVC C4127 "conditional expression is constant" triggered by +// RETURN_CUDNN_FRONTEND_ERROR_IF macro (which has `if (retval == error_code_t::OK)`) +#if defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable : 4127) +#endif + +#include +#include + +namespace cudnn_frontend { +namespace experimental { + +// ============================================================ +// Kernel spec lookup for SM100 SDPA prefill kernels +// ============================================================ + +/// @brief Look up the SM100 kernel specification for a given head dimension. +/// @param d Head dimension (d_qk). Currently supports 64 and 128. +/// @param sm_version SM version from cudaDeviceProp (e.g. 100, 101). +/// @return Pointer to static KernelSpec, or nullptr if unsupported. +inline const KernelSpec* +lookup_sm100_kernel_spec(int d, int sm_version) { + // SM100 family: sm_version 100..109 + if (sm_version / 10 != 10) return nullptr; + + static const KernelSpec spec_d128 = { + generated::sm100_d128_fprop_source, + generated::sm100_d128_fprop_source_len, + generated::sm100_d128_fprop_flags, + generated::sm100_d128_fprop_flags_len, + "cudnn_generated_oss_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_cga1x1x1_kernel0_0", + 128, + 128, + 128, + 232448 // smem_bytes for d=128 + }; + + static const KernelSpec spec_d64 = { + generated::sm100_d64_fprop_source, + generated::sm100_d64_fprop_source_len, + generated::sm100_d64_fprop_flags, + generated::sm100_d64_fprop_flags_len, + "cudnn_generated_oss_sdpa_sm100_flash_fprop_f16_knob_7_128x128x64_4x1x1_cga1x1x1_kernel0_0", + 128, + 128, + 64, + 232448 // smem_bytes for d=64 + }; + + switch (d) { + case 128: + return &spec_d128; + case 64: + return &spec_d64; + default: + return nullptr; + } +} + +// ============================================================ +// Sm100SdpaPrefillEngine +// ============================================================ + +/// @brief SM100-specific SDPA prefill engine using NVRTC-compiled kernels. +/// +/// This engine targets the SM100 (Blackwell) architecture and uses L2-aware +/// tile scheduling instead of the SM90 persistent-tile-counter approach. +/// The kernel is compiled at runtime via NVRTC from embedded source and +/// launched with TMA descriptors for Q, K, V, and O tensors. +/// +/// Usage: +/// 1. check_support() -- validates shape + SM version, caches device info. +/// 2. build() -- NVRTC-compiles the kernel and loads the CU module. +/// 3. execute() -- sets up TMA descriptors, packs params, launches. +/// +/// Thread safety: instances are NOT thread-safe. Each thread should own its +/// own engine, or external synchronization must be provided. +class Sm100SdpaPrefillEngine : public IOssSdpaEngine { + public: + Sm100SdpaPrefillEngine() = default; + + ~Sm100SdpaPrefillEngine() override { + if (module_) { + detail::cuda_library_unload(module_); + } + } + + // Non-copyable + Sm100SdpaPrefillEngine(const Sm100SdpaPrefillEngine&) = delete; + Sm100SdpaPrefillEngine& + operator=(const Sm100SdpaPrefillEngine&) = delete; + + // Movable + Sm100SdpaPrefillEngine(Sm100SdpaPrefillEngine&& other) noexcept + : spec_(other.spec_), + d_(other.d_), + sm_version_(other.sm_version_), + sm_count_(other.sm_count_), + support_checked_(other.support_checked_), + module_(other.module_), + kernelPtr_(other.kernelPtr_), + cubin_(std::move(other.cubin_)), + cubinSize_(other.cubinSize_), + tile_m_(other.tile_m_), + tile_n_(other.tile_n_), + tile_k_(other.tile_k_), + smemBytes_(other.smemBytes_), + kernelName_(std::move(other.kernelName_)), + built_(other.built_) { + other.module_ = nullptr; + other.kernelPtr_ = nullptr; + other.built_ = false; + } + + Sm100SdpaPrefillEngine& + operator=(Sm100SdpaPrefillEngine&& other) noexcept { + if (this != &other) { + if (module_) { + detail::cuda_library_unload(module_); + } + spec_ = other.spec_; + d_ = other.d_; + sm_version_ = other.sm_version_; + sm_count_ = other.sm_count_; + support_checked_ = other.support_checked_; + module_ = other.module_; + kernelPtr_ = other.kernelPtr_; + cubin_ = std::move(other.cubin_); + cubinSize_ = other.cubinSize_; + tile_m_ = other.tile_m_; + tile_n_ = other.tile_n_; + tile_k_ = other.tile_k_; + smemBytes_ = other.smemBytes_; + kernelName_ = std::move(other.kernelName_); + built_ = other.built_; + other.module_ = nullptr; + other.kernelPtr_ = nullptr; + other.built_ = false; + } + return *this; + } + + // ---- Phase 1: Support check (spec lookup, cache SM count) ---- + + /// @brief Validate that the given attention shape and SM version are supported. + /// @param shape Attention shape (batch, heads, seqlens, head dim). + /// @param sm_version SM version (e.g. 100 for SM100). + /// @return error_t with OK on success, GRAPH_NOT_SUPPORTED if unsupported. + error_t + check_support(AttentionShape_t shape, int sm_version) override { + spec_ = lookup_sm100_kernel_spec(shape.d_qk, sm_version); + RETURN_CUDNN_FRONTEND_ERROR_IF( + spec_ == nullptr, + error_code_t::GRAPH_NOT_SUPPORTED, + "Unsupported config: d=" + std::to_string(shape.d_qk) + " sm_version=" + std::to_string(sm_version)); + + d_ = shape.d_qk; + sm_version_ = sm_version; + tile_m_ = spec_->tile_m; + tile_n_ = spec_->tile_n; + tile_k_ = spec_->tile_k; + smemBytes_ = spec_->smem_bytes; + kernelName_ = spec_->kernel_name; + + // Cache SM count for later use in execute() + cudaDeviceProp devProp; + int device_ordinal = 0; + cudaError_t cuda_err = detail::cuda_get_device(&device_ordinal); + RETURN_CUDNN_FRONTEND_ERROR_IF(cuda_err != cudaSuccess, error_code_t::CUDA_API_FAILED, "cudaGetDevice failed"); + cuda_err = detail::cuda_get_device_properties(&devProp, device_ordinal); + RETURN_CUDNN_FRONTEND_ERROR_IF( + cuda_err != cudaSuccess, error_code_t::CUDA_API_FAILED, "cudaGetDeviceProperties failed"); + sm_count_ = devProp.multiProcessorCount; + + support_checked_ = true; + return {error_code_t::OK, ""}; + } + + /// @brief Returns the workspace size in bytes needed by execute(). + /// SM100 does not use a workspace tile counter, but we keep the interface + /// consistent with SM90 for uniform allocation by the caller. + static int64_t + get_workspace_size() { + return 16; + } + + // ---- Phase 2: NVRTC compilation + module loading ---- + + /// @brief Compile the kernel via NVRTC and load the resulting CU module. + /// @pre check_support() must have been called successfully. + /// @return error_t with OK on success. + error_t + build() override { + RETURN_CUDNN_FRONTEND_ERROR_IF(!support_checked_ || spec_ == nullptr, + error_code_t::INVALID_VALUE, + "build() called before check_support()"); + + auto status = compile_and_load_kernel(spec_, module_, kernelPtr_, cubin_, cubinSize_, kernelName_); + if (status.is_good()) { + built_ = true; + } + return status; + } + + // ---- Phase 3: Set up TMA descriptors, pack params, launch ---- + + /// @brief Execute the SM100 SDPA prefill kernel. + /// + /// Sets up TMA descriptors for Q/K/V/O, computes L2-aware tile scheduling + /// parameters, and launches the kernel on the given stream. + /// + /// Strides are in ELEMENTS (not bytes), matching PyTorch's .stride() convention. + /// All stride vectors have 4 elements in (b, h, s, d) order. + /// For max/sum_exp the 4th element (d-stride) is typically 1 and is unused internally. + /// + /// @param batch Batch size (B). + /// @param heads_q Number of query heads (H_q). + /// @param heads_kv Number of key/value heads (H_kv). + /// @param seq_q Query sequence length (S_q). + /// @param seq_kv Key/value sequence length (S_kv). + /// @param d Head dimension (d_qk = d_v). + /// @param d_Q Device pointer to Q tensor (fp16). + /// @param q_strides Q strides in elements: [b, h, s, d]. + /// @param d_K Device pointer to K tensor (fp16). + /// @param k_strides K strides in elements: [b, h, s, d]. + /// @param d_V Device pointer to V tensor (fp16). + /// @param v_strides V strides in elements: [b, h, s, d]. + /// @param d_O Device pointer to O tensor (fp16, output). + /// @param o_strides O strides in elements: [b, h, s, d]. + /// @param d_max Device pointer to per-row max values (fp32, [B, H_q, S_q]). + /// @param max_strides Max strides in elements: [b, h, s]. + /// @param d_sum_exp Device pointer to per-row sum-of-exp values (fp32, [B, H_q, S_q]). + /// @param se_strides Sum-exp strides in elements: [b, h, s]. + /// @param workspace Device workspace pointer (at least 16 bytes, unused by SM100 but kept for API consistency). + /// @param device Device ordinal for kernel attribute setting. + /// @param stream CUDA stream for asynchronous execution. + /// @param user_attn_scale Optional attention scale; defaults to 1/sqrt(D). + /// @return error_t with OK on success. + error_t + execute(int batch, + int heads_q, + int heads_kv, + int seq_q, + int seq_kv, + int d, + void* d_Q, + std::vector const& q_strides, + void* d_K, + std::vector const& k_strides, + void* d_V, + std::vector const& v_strides, + void* d_O, + std::vector const& o_strides, + void* d_max, + std::vector const& max_strides, + void* d_sum_exp, + std::vector const& se_strides, + void* workspace, + int device, + cudaStream_t stream, + std::optional user_attn_scale = std::nullopt) override { + RETURN_CUDNN_FRONTEND_ERROR_IF(!built_, error_code_t::INVALID_VALUE, "execute() called before build()"); + RETURN_CUDNN_FRONTEND_ERROR_IF( + d > 128, error_code_t::GRAPH_NOT_SUPPORTED, "SM100 OSS engine requires d <= 128"); + RETURN_CUDNN_FRONTEND_ERROR_IF( + q_strides.size() < 4 || k_strides.size() < 4 || v_strides.size() < 4 || o_strides.size() < 4, + error_code_t::INVALID_VALUE, + "Q/K/V/O strides must have at least 4 elements"); + RETURN_CUDNN_FRONTEND_ERROR_IF(max_strides.size() < 3 || se_strides.size() < 3, + error_code_t::INVALID_VALUE, + "max/sum_exp strides must have at least 3 elements"); + + // Suppress unused-parameter warning for workspace (SM100 does not use a tile counter) + (void)workspace; + + int B = batch; + int H_q = heads_q; + int H_kv = heads_kv; + int S_q = seq_q; + int S_kv = seq_kv; + int D = d; + int TILE_M = tile_m_; + int TILE_N = tile_n_; + + // ---- Set up AttentionDescriptor ---- + AttentionDescriptor_t attnDesc = {}; + attnDesc.b = B; + attnDesc.q_h = H_q; + attnDesc.k_h = H_kv; + attnDesc.v_h = H_kv; + attnDesc.s_q = S_q; + attnDesc.s_kv = S_kv; + attnDesc.d_qk = D; + attnDesc.d_v = D; + attnDesc.q_heads_per_k = (H_q >= H_kv && H_kv > 0) ? (uint16_t)(H_q / H_kv) : 1; + attnDesc.q_heads_per_v = (H_q >= H_kv && H_kv > 0) ? (uint16_t)(H_q / H_kv) : 1; + attnDesc.min_q_heads_per_kv = attnDesc.q_heads_per_k; + + // ---- L2-aware tile scheduling ---- + // SM100 uses an L2-aware swizzle pattern for better cache utilization + // instead of SM90's persistent tile counter approach. + constexpr int Tiles_Q = 2; + constexpr uint32_t size_l2 = 50u * 1024u * 1024u; + + int ctas_in_q_dim = div_up(S_q, TILE_M * Tiles_Q); + int num_hb = B * H_q; + + // Compute the swizzle factor based on how many KV heads fit in L2 + uint32_t one_kv_head_size = static_cast(S_kv) * (D + D) * sizeof(half); + int swizzle = (one_kv_head_size > 0) ? (1 << find_log_2_floor(size_l2 / one_kv_head_size)) : 1; + swizzle = std::max(1, std::min(swizzle, num_hb)); + + int num_hb_quotient = num_hb / swizzle; + int num_hb_remainder = num_hb % swizzle; + int num_block = ctas_in_q_dim; + int total_blocks = num_block * num_hb; + + // Fast divisors for tile index decomposition in the kernel + FastDivisor_t num_head_divmod = make_fast_divisor(static_cast(H_q)); + FastDivisor_t l2_minor_divmod = make_fast_divisor(static_cast(swizzle)); + FastDivisor_t l2_major_divmod = make_fast_divisor(static_cast(swizzle * num_block)); + FastDivisor_t l2_minor_residual_divmod = + make_fast_divisor(static_cast(std::max(1, num_hb_remainder))); + + // ---- TMA descriptors (4D) ---- + // Caller strides are in elements; TMA needs bytes. + uint32_t elem_bytes = sizeof(half); + uint32_t elems_per_ld = std::min(64, D); + + // Q TMA: dims = (D, S_q, H_q, B), box = (elems_per_ld, TILE_M) + // SM100 uses TILE_M (128) directly, unlike SM90 which uses TILE_M * 2. + CUtensorMap tma_q = {}; + CHECK_CUDNN_FRONTEND_ERROR(create_tma_desc_4d(&tma_q, + d_Q, + CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, + D, + S_q, + H_q, + B, + (uint64_t)q_strides[2] * elem_bytes, + (uint64_t)q_strides[1] * elem_bytes, + (uint64_t)q_strides[0] * elem_bytes, + elems_per_ld, + TILE_M, + CU_TENSOR_MAP_SWIZZLE_128B)); + + // K TMA: dims = (D, S_kv, H_kv, B), box = (elems_per_ld, TILE_N) + CUtensorMap tma_k = {}; + CHECK_CUDNN_FRONTEND_ERROR(create_tma_desc_4d(&tma_k, + d_K, + CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, + D, + S_kv, + H_kv, + B, + (uint64_t)k_strides[2] * elem_bytes, + (uint64_t)k_strides[1] * elem_bytes, + (uint64_t)k_strides[0] * elem_bytes, + elems_per_ld, + TILE_N, + CU_TENSOR_MAP_SWIZZLE_128B)); + + // V TMA: dims = (D, S_kv, H_kv, B), box = (elems_per_ld, TILE_N) + CUtensorMap tma_v = {}; + CHECK_CUDNN_FRONTEND_ERROR(create_tma_desc_4d(&tma_v, + d_V, + CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, + D, + S_kv, + H_kv, + B, + (uint64_t)v_strides[2] * elem_bytes, + (uint64_t)v_strides[1] * elem_bytes, + (uint64_t)v_strides[0] * elem_bytes, + elems_per_ld, + TILE_N, + CU_TENSOR_MAP_SWIZZLE_128B)); + + // O TMA: dims = (D, S_q, H_q, B), box = (elems_per_ld, TILE_M) + // SM100 uses TILE_M (128) for the O box dim, unlike SM90 which uses 16. + CUtensorMap tma_o = {}; + CHECK_CUDNN_FRONTEND_ERROR(create_tma_desc_4d(&tma_o, + d_O, + CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, + D, + S_q, + H_q, + B, + (uint64_t)o_strides[2] * elem_bytes, + (uint64_t)o_strides[1] * elem_bytes, + (uint64_t)o_strides[0] * elem_bytes, + elems_per_ld, + TILE_M, + CU_TENSOR_MAP_SWIZZLE_128B)); + + // ---- Scaling ---- + float attn_scale = user_attn_scale.value_or(1.0f / std::sqrt((float)D)); + float tensor_13 = -INFINITY; + + // ---- Tensor descriptors for max and sum_exp (f32, [B, H_q, S_q]) ---- + // Strides provided by caller in elements. + tensor_descriptor desc_max = {}; + desc_max.num_dims = 3; + desc_max.dims[0] = B; + desc_max.dims[1] = H_q; + desc_max.dims[2] = S_q; + desc_max.strides[0] = max_strides[0]; + desc_max.strides[1] = max_strides[1]; + desc_max.strides[2] = max_strides[2]; + + tensor_descriptor desc_sum_exp = {}; + desc_sum_exp.num_dims = 3; + desc_sum_exp.dims[0] = B; + desc_sum_exp.dims[1] = H_q; + desc_sum_exp.dims[2] = S_q; + desc_sum_exp.strides[0] = se_strides[0]; + desc_sum_exp.strides[1] = se_strides[1]; + desc_sum_exp.strides[2] = se_strides[2]; + + // ---- Grid/Block ---- + // SM100 launches total_blocks CTAs with 512 threads each. + // No persistent scheduling -- the L2 swizzle parameters handle work distribution. + dim3 grid(total_blocks, 1, 1); + dim3 block(512, 1, 1); + + // ---- Build kernel params array (18 parameters) ---- + // The parameter order must match the SM100 kernel's expected layout exactly. + void* kernelParams[] = { + (void*)&attnDesc, + (void*)&num_head_divmod, + (void*)&l2_minor_divmod, + (void*)&l2_major_divmod, + (void*)&l2_minor_residual_divmod, + (void*)&num_hb_quotient, + (void*)&num_block, + (void*)&total_blocks, + (void*)&tma_q, + (void*)&tma_k, + (void*)&attn_scale, + (void*)&tensor_13, + (void*)&d_max, + (void*)&desc_max, + (void*)&d_sum_exp, + (void*)&desc_sum_exp, + (void*)&tma_v, + (void*)&tma_o, + }; + + // ---- Debug logging ---- + if (cudnn_frontend::isLoggingEnabled()) { + CUDNN_FE_LOG_LABEL_ENDL("SM100 OSS Engine Launch:"); + CUDNN_FE_LOG_LABEL_ENDL(" Grid = (" << grid.x << "," << grid.y << "," << grid.z << ")"); + CUDNN_FE_LOG_LABEL_ENDL(" Block = (" << block.x << "," << block.y << "," << block.z << ")"); + CUDNN_FE_LOG_LABEL_ENDL(" Smem = " << smemBytes_); + CUDNN_FE_LOG_LABEL_ENDL(" Kernel: " << kernelName_); + CUDNN_FE_LOG_LABEL_ENDL(" B=" << B << " H_q=" << H_q << " H_kv=" << H_kv << " S_q=" << S_q + << " S_kv=" << S_kv << " D=" << D); + CUDNN_FE_LOG_LABEL_ENDL(" L2: swizzle=" << swizzle << " num_hb_quotient=" << num_hb_quotient + << " num_block=" << num_block << " total_blocks=" << total_blocks); + CUDNN_FE_LOG_LABEL_ENDL(" attn_scale=" << attn_scale); + } + + // ---- Set shared memory attribute and launch ---- + cudaError_t cuda_err; + + cuda_err = detail::cuda_kernel_set_attribute_for_device( + kernelPtr_, cudaFuncAttributeMaxDynamicSharedMemorySize, smemBytes_, device); + RETURN_CUDNN_FRONTEND_ERROR_IF(cuda_err != cudaSuccess, + error_code_t::CUDA_API_FAILED, + "cudaKernelSetAttributeForDevice failed (smem=" + std::to_string(smemBytes_) + + " bytes, error=" + detail::cuda_error_to_string(cuda_err) + ")"); + + cuda_err = detail::cuda_launch_kernel((const void*)kernelPtr_, grid, block, kernelParams, smemBytes_, stream); + cudaError_t last_err = detail::cuda_get_last_error(); + RETURN_CUDNN_FRONTEND_ERROR_IF(cuda_err != cudaSuccess, + error_code_t::CUDA_API_FAILED, + "cudaLaunchKernel failed (grid=" + std::to_string(grid.x) + + ", block=512, smem=" + std::to_string(smemBytes_) + + ", error=" + detail::cuda_error_to_string(cuda_err) + ")"); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + last_err != cudaSuccess, + error_code_t::CUDA_API_FAILED, + "cudaGetLastError after kernel launch: " + detail::cuda_error_to_string(last_err)); + + return {error_code_t::OK, ""}; + } + + private: + // State from check_support() + const KernelSpec* spec_ = nullptr; + int d_ = 0; + int sm_version_ = 0; + int sm_count_ = 0; + bool support_checked_ = false; + + // State from build() + cudaLibrary_t module_ = nullptr; + cudaKernel_t kernelPtr_ = nullptr; + std::unique_ptr cubin_; + size_t cubinSize_ = 0; + int tile_m_ = 0, tile_n_ = 0, tile_k_ = 0; + int smemBytes_ = 0; + std::string kernelName_; + bool built_ = false; +}; + +} // namespace experimental +} // namespace cudnn_frontend + +#if defined(_MSC_VER) +#pragma warning(pop) +#endif diff --git a/deps/cudnn-frontend/include/cudnn_frontend/experimental/sm90_sdpa_prefill_engine.h b/deps/cudnn-frontend/include/cudnn_frontend/experimental/sm90_sdpa_prefill_engine.h new file mode 100644 index 0000000..d1fb8a2 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/experimental/sm90_sdpa_prefill_engine.h @@ -0,0 +1,450 @@ +/* + * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#pragma once + +#include "oss_engine_interface.h" +#include "../generated/sdpa/sm90/prefill/full_seqlens/d128_fprop_kernel.h" +#include "../generated/sdpa/sm90/prefill/full_seqlens/d64_fprop_kernel.h" + +#include + +// Suppress MSVC C4127 "conditional expression is constant" triggered by +// RETURN_CUDNN_FRONTEND_ERROR_IF macro (which has `if (retval == error_code_t::OK)`) +#if defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable : 4127) +#endif + +#include +#include + +namespace cudnn_frontend { +namespace experimental { + +// ============================================================ +// Kernel spec lookup (replaces KernelFactory) +// ============================================================ + +inline const KernelSpec* +lookup_kernel_spec(int d, int sm_version) { + if (sm_version != 90) return nullptr; + + static const KernelSpec spec_d128 = { + generated::d128_fprop_source, + generated::d128_fprop_source_len, + generated::d128_fprop_flags, + generated::d128_fprop_flags_len, + "cudnn_generated_oss_sdpa_sm90_flash_fprop_wgmma_f16_knob_7_64x128x128_4x1x1_cga1x1x1_kernel0_0", + 64, + 128, + 128, + 232448 // smem_bytes for d=128 (227 * 1024) + }; + + static const KernelSpec spec_d64 = { + generated::d64_fprop_source, + generated::d64_fprop_source_len, + generated::d64_fprop_flags, + generated::d64_fprop_flags_len, + "cudnn_generated_oss_sdpa_sm90_flash_fprop_wgmma_f16_knob_7_64x128x64_4x1x1_cga1x1x1_kernel0_0", + 64, + 128, + 64, + 122880 // smem_bytes for d=64 (120 * 1024) + }; + + switch (d) { + case 128: + return &spec_d128; + case 64: + return &spec_d64; + default: + return nullptr; + } +} +// ============================================================ +// Sm90SdpaPrefillEngine +// ============================================================ + +class Sm90SdpaPrefillEngine : public IOssSdpaEngine { + public: + Sm90SdpaPrefillEngine() = default; + + ~Sm90SdpaPrefillEngine() override { + if (module_) { + detail::cuda_library_unload(module_); + } + } + + // Non-copyable + Sm90SdpaPrefillEngine(const Sm90SdpaPrefillEngine&) = delete; + Sm90SdpaPrefillEngine& + operator=(const Sm90SdpaPrefillEngine&) = delete; + + // Movable + Sm90SdpaPrefillEngine(Sm90SdpaPrefillEngine&& other) noexcept + : spec_(other.spec_), + d_(other.d_), + sm_version_(other.sm_version_), + sm_count_(other.sm_count_), + support_checked_(other.support_checked_), + module_(other.module_), + kernelPtr_(other.kernelPtr_), + cubin_(std::move(other.cubin_)), + cubinSize_(other.cubinSize_), + tile_m_(other.tile_m_), + tile_n_(other.tile_n_), + tile_k_(other.tile_k_), + smemBytes_(other.smemBytes_), + kernelName_(std::move(other.kernelName_)), + built_(other.built_) { + other.module_ = nullptr; + other.kernelPtr_ = nullptr; + other.built_ = false; + } + + Sm90SdpaPrefillEngine& + operator=(Sm90SdpaPrefillEngine&& other) noexcept { + if (this != &other) { + if (module_) { + detail::cuda_library_unload(module_); + } + spec_ = other.spec_; + d_ = other.d_; + sm_version_ = other.sm_version_; + sm_count_ = other.sm_count_; + support_checked_ = other.support_checked_; + module_ = other.module_; + kernelPtr_ = other.kernelPtr_; + cubin_ = std::move(other.cubin_); + cubinSize_ = other.cubinSize_; + tile_m_ = other.tile_m_; + tile_n_ = other.tile_n_; + tile_k_ = other.tile_k_; + smemBytes_ = other.smemBytes_; + kernelName_ = std::move(other.kernelName_); + built_ = other.built_; + other.module_ = nullptr; + other.kernelPtr_ = nullptr; + other.built_ = false; + } + return *this; + } + + // ---- Phase 1: Support check (spec lookup, cache SM count) ---- + error_t + check_support(AttentionShape_t shape, int sm_version) override { + spec_ = lookup_kernel_spec(shape.d_qk, sm_version); + RETURN_CUDNN_FRONTEND_ERROR_IF( + spec_ == nullptr, + error_code_t::GRAPH_NOT_SUPPORTED, + "Unsupported config: d=" + std::to_string(shape.d_qk) + " sm_version=" + std::to_string(sm_version)); + + d_ = shape.d_qk; + sm_version_ = sm_version; + tile_m_ = spec_->tile_m; + tile_n_ = spec_->tile_n; + tile_k_ = spec_->tile_k; + smemBytes_ = spec_->smem_bytes; + kernelName_ = spec_->kernel_name; + + // Cache SM count for later use in execute() + cudaDeviceProp devProp; + int device_ordinal = 0; + cudaError_t cuda_err = detail::cuda_get_device(&device_ordinal); + RETURN_CUDNN_FRONTEND_ERROR_IF(cuda_err != cudaSuccess, error_code_t::CUDA_API_FAILED, "cudaGetDevice failed"); + cuda_err = detail::cuda_get_device_properties(&devProp, device_ordinal); + RETURN_CUDNN_FRONTEND_ERROR_IF( + cuda_err != cudaSuccess, error_code_t::CUDA_API_FAILED, "cudaGetDeviceProperties failed"); + sm_count_ = devProp.multiProcessorCount; + + support_checked_ = true; + return {error_code_t::OK, ""}; + } + + // Workspace size needed by execute() (tile_id_counter: 4 bytes, 16-byte aligned) + static int64_t + get_workspace_size() { + return 16; + } + + // ---- Phase 2: NVRTC compilation + module loading ---- + error_t + build() override { + RETURN_CUDNN_FRONTEND_ERROR_IF(!support_checked_ || spec_ == nullptr, + error_code_t::INVALID_VALUE, + "build() called before check_support()"); + + auto status = compile_and_load_kernel(spec_, module_, kernelPtr_, cubin_, cubinSize_, kernelName_); + if (status.is_good()) { + built_ = true; + } + return status; + } + + // ---- Phase 3: Set up TMA descriptors, pack params, launch ---- + // Strides are in ELEMENTS (not bytes), matching PyTorch's .stride() convention. + // All stride vectors have 4 elements in (b, h, s, d) order. + // For max/sum_exp the 4th element (d-stride) is typically 1 and is unused internally. + error_t + execute(int batch, + int heads_q, + int heads_kv, + int seq_q, + int seq_kv, + int d, + void* d_Q, + std::vector const& q_strides, + void* d_K, + std::vector const& k_strides, + void* d_V, + std::vector const& v_strides, + void* d_O, + std::vector const& o_strides, + void* d_max, + std::vector const& max_strides, + void* d_sum_exp, + std::vector const& se_strides, + void* workspace, + int device, + cudaStream_t stream, + std::optional user_attn_scale = std::nullopt) override { + RETURN_CUDNN_FRONTEND_ERROR_IF(!built_, error_code_t::INVALID_VALUE, "execute() called before build()"); + RETURN_CUDNN_FRONTEND_ERROR_IF( + q_strides.size() < 4 || k_strides.size() < 4 || v_strides.size() < 4 || o_strides.size() < 4, + error_code_t::INVALID_VALUE, + "Q/K/V/O strides must have at least 4 elements"); + RETURN_CUDNN_FRONTEND_ERROR_IF(max_strides.size() < 3 || se_strides.size() < 3, + error_code_t::INVALID_VALUE, + "max/sum_exp strides must have at least 3 elements"); + + int B = batch; + int H_q = heads_q; + int H_kv = heads_kv; + int S_q = seq_q; + int S_kv = seq_kv; + int D = d; + int TILE_M = tile_m_; + int TILE_N = tile_n_; + + // ---- Set up AttentionDescriptor ---- + AttentionDescriptor_t attnDesc = {}; + attnDesc.b = B; + attnDesc.q_h = H_q; + attnDesc.k_h = H_kv; + attnDesc.v_h = H_kv; + attnDesc.s_q = S_q; + attnDesc.s_kv = S_kv; + attnDesc.d_qk = D; + attnDesc.d_v = D; + attnDesc.q_heads_per_k = (H_q >= H_kv && H_kv > 0) ? (uint16_t)(H_q / H_kv) : 1; + attnDesc.q_heads_per_v = (H_q >= H_kv && H_kv > 0) ? (uint16_t)(H_q / H_kv) : 1; + attnDesc.min_q_heads_per_kv = attnDesc.q_heads_per_k; + + // ---- Tile scheduling ---- + int effective_tile_m = TILE_M * 2; + int tiles_r = div_up(S_q, effective_tile_m); + int tiles_hr = H_q * tiles_r; + int num_tiles = B * tiles_hr; + FastDivisor_t tiles_hr_div = make_fast_divisor(tiles_hr); + FastDivisor_t tiles_r_div = make_fast_divisor(tiles_r); + + // ---- TMA descriptors (4D) ---- + // Caller strides are in elements; TMA needs bytes. + uint32_t elem_bytes = sizeof(half); + uint32_t elems_per_ld = std::min(64, D); + + // Q TMA (tma_tensor_3): dims = (D, S_q, H_q, B) + CUtensorMap tma_q = {}; + CHECK_CUDNN_FRONTEND_ERROR(create_tma_desc_4d(&tma_q, + d_Q, + CU_TENSOR_MAP_DATA_TYPE_FLOAT16, + D, + S_q, + H_q, + B, + (uint64_t)q_strides[2] * elem_bytes, + (uint64_t)q_strides[1] * elem_bytes, + (uint64_t)q_strides[0] * elem_bytes, + elems_per_ld, + TILE_M * 2, + CU_TENSOR_MAP_SWIZZLE_128B)); + + // K TMA (tma_tensor_2) + CUtensorMap tma_k = {}; + CHECK_CUDNN_FRONTEND_ERROR(create_tma_desc_4d(&tma_k, + d_K, + CU_TENSOR_MAP_DATA_TYPE_FLOAT16, + D, + S_kv, + H_kv, + B, + (uint64_t)k_strides[2] * elem_bytes, + (uint64_t)k_strides[1] * elem_bytes, + (uint64_t)k_strides[0] * elem_bytes, + elems_per_ld, + TILE_N, + CU_TENSOR_MAP_SWIZZLE_128B)); + + // V TMA (tma_tensor_1) + CUtensorMap tma_v = {}; + CHECK_CUDNN_FRONTEND_ERROR(create_tma_desc_4d(&tma_v, + d_V, + CU_TENSOR_MAP_DATA_TYPE_FLOAT16, + D, + S_kv, + H_kv, + B, + (uint64_t)v_strides[2] * elem_bytes, + (uint64_t)v_strides[1] * elem_bytes, + (uint64_t)v_strides[0] * elem_bytes, + elems_per_ld, + TILE_N, + CU_TENSOR_MAP_SWIZZLE_128B)); + + // O TMA (tma_tensor_5) + CUtensorMap tma_o = {}; + CHECK_CUDNN_FRONTEND_ERROR(create_tma_desc_4d(&tma_o, + d_O, + CU_TENSOR_MAP_DATA_TYPE_FLOAT16, + D, + S_q, + H_q, + B, + (uint64_t)o_strides[2] * elem_bytes, + (uint64_t)o_strides[1] * elem_bytes, + (uint64_t)o_strides[0] * elem_bytes, + elems_per_ld, + 16, + CU_TENSOR_MAP_SWIZZLE_128B)); + + // ---- Scaling ---- + float attn_scale = user_attn_scale.value_or(1.0f / std::sqrt((float)D)); + float tensor_13 = -INFINITY; + + // ---- Tensor descriptors for max and sum_exp (f32, [B, H_q, S_q]) ---- + // Strides provided by caller in elements. + tensor_descriptor desc_max = {}; + desc_max.num_dims = 3; + desc_max.dims[0] = B; + desc_max.dims[1] = H_q; + desc_max.dims[2] = S_q; + desc_max.strides[0] = max_strides[0]; + desc_max.strides[1] = max_strides[1]; + desc_max.strides[2] = max_strides[2]; + + tensor_descriptor desc_sum_exp = {}; + desc_sum_exp.num_dims = 3; + desc_sum_exp.dims[0] = B; + desc_sum_exp.dims[1] = H_q; + desc_sum_exp.dims[2] = S_q; + desc_sum_exp.strides[0] = se_strides[0]; + desc_sum_exp.strides[1] = se_strides[1]; + desc_sum_exp.strides[2] = se_strides[2]; + + // ---- Grid/Block ---- + int CTAs = std::min(sm_count_, num_tiles); + dim3 grid(CTAs, 1, 1); + dim3 block(384, 1, 1); + + // ---- Memset tile_id_counter (uses caller-provided workspace) ---- + RETURN_CUDNN_FRONTEND_ERROR_IF(!workspace, + error_code_t::INVALID_VALUE, + "workspace must not be null (need at least 16 bytes for tile_id_counter)"); + int32_t* d_tile_id_counter = reinterpret_cast(workspace); + { + cudaError_t memset_err = detail::cuda_mem_set_async(d_tile_id_counter, 0, sizeof(int32_t), stream); + RETURN_CUDNN_FRONTEND_ERROR_IF( + memset_err != cudaSuccess, error_code_t::CUDA_API_FAILED, "cudaMemsetAsync for tile_id_counter failed"); + } + + // ---- Build kernel params array ---- + void* kernelParams[] = { + (void*)&attnDesc, + (void*)&num_tiles, + (void*)&tiles_hr_div, + (void*)&tiles_r_div, + (void*)&d_tile_id_counter, + (void*)&tma_q, + (void*)&tma_k, + (void*)&attn_scale, + (void*)&tensor_13, + (void*)&d_max, + (void*)&desc_max, + (void*)&d_sum_exp, + (void*)&desc_sum_exp, + (void*)&tma_v, + (void*)&tma_o, + }; + + // ---- Set shared memory attribute and launch ---- + cudaError_t cuda_err; + + cuda_err = detail::cuda_kernel_set_attribute_for_device( + kernelPtr_, cudaFuncAttributeMaxDynamicSharedMemorySize, smemBytes_, device); + RETURN_CUDNN_FRONTEND_ERROR_IF(cuda_err != cudaSuccess, + error_code_t::CUDA_API_FAILED, + "cudaKernelSetAttributeForDevice failed (smem=" + std::to_string(smemBytes_) + + " bytes, error=" + detail::cuda_error_to_string(cuda_err) + ")"); + + cuda_err = detail::cuda_launch_kernel((const void*)kernelPtr_, grid, block, kernelParams, smemBytes_, stream); + cudaError_t last_err = detail::cuda_get_last_error(); + RETURN_CUDNN_FRONTEND_ERROR_IF(cuda_err != cudaSuccess, + error_code_t::CUDA_API_FAILED, + "cudaLaunchKernel failed (grid=" + std::to_string(grid.x) + + ", block=384, smem=" + std::to_string(smemBytes_) + + ", error=" + detail::cuda_error_to_string(cuda_err) + ")"); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + last_err != cudaSuccess, + error_code_t::CUDA_API_FAILED, + "cudaGetLastError after kernel launch: " + detail::cuda_error_to_string(last_err)); + + return {error_code_t::OK, ""}; + } + + private: + // State from check_support() + const KernelSpec* spec_ = nullptr; + int d_ = 0; + int sm_version_ = 0; + int sm_count_ = 0; + bool support_checked_ = false; + + // State from build() + cudaLibrary_t module_ = nullptr; + cudaKernel_t kernelPtr_ = nullptr; + std::unique_ptr cubin_; + size_t cubinSize_ = 0; + int tile_m_ = 0, tile_n_ = 0, tile_k_ = 0; + int smemBytes_ = 0; + std::string kernelName_; + bool built_ = false; +}; + +} // namespace experimental +} // namespace cudnn_frontend + +#if defined(_MSC_VER) +#pragma warning(pop) +#endif diff --git a/deps/cudnn-frontend/include/cudnn_frontend/generated/rms_norm_silu/sm100/ln_fwd_silu_kernel.h b/deps/cudnn-frontend/include/cudnn_frontend/generated/rms_norm_silu/sm100/ln_fwd_silu_kernel.h new file mode 100644 index 0000000..77b5dd7 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/generated/rms_norm_silu/sm100/ln_fwd_silu_kernel.h @@ -0,0 +1,468 @@ +// Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: MIT +#pragma once + +// Auto-generated from ln_fwd_silu_kernel.cu +// Do not edit manually. + +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Woverlength-strings" +#endif + +#if defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable : 4068) +#endif + +namespace cudnn_frontend::experimental::generated { + +inline constexpr const char ln_fwd_silu_kernel_source[] = + R"KERNEL( +constexpr int mxfp8_block_size = 32; +constexpr int nvfp4_block_size = 16; + +template +struct LnFwdShared { + using Traits = _Traits; + static constexpr int32_t SMEM_STATS_ELEMENTS = ((Traits::Stats::SMEM_BYTES > 0) ? Traits::Stats::SMEM_BYTES : 1); + static constexpr int32_t SMEM_BAR_ELEMENTS = ((Traits::USE_CLUSTER && Traits::CTAS_PER_ROW > 1) + ? (Traits::WARPS_M + 1 + Traits::WARPS_M * Traits::CTAS_PER_ROW) + : 1); + static constexpr int32_t SMEM_MXFP8_ELEMENTS = + ((isBlockScale_1D2X2X && !useBlockScaleColwiseKernel) + ? ((mxfp8_block_size * Traits::NUM_ELTS + (Traits::NUM_ELTS - 1)) * (mxfp8_block_size + 1)) + : 1); + static constexpr int32_t GAMMA_ELEMENTS = + ((Traits::hasGamma && Traits::USE_GAMMA_SMEM) + ? (Traits::BATCH_SIZE * Traits::LDGS * Traits::THREADS_PER_ROW * Traits::NUM_ELTS) + : 1); + static constexpr int32_t BETA_ELEMENTS = + ((Traits::hasBeta && Traits::USE_GAMMA_SMEM) + ? (Traits::BATCH_SIZE * Traits::LDGS * Traits::THREADS_PER_ROW * Traits::NUM_ELTS) + : 1); + + __align__(16) char smem_stats[SMEM_STATS_ELEMENTS]; + __align__(16) uint64_t smem_bar[SMEM_BAR_ELEMENTS]; + __align__(16) typename Traits::weight_t smem_gamma[GAMMA_ELEMENTS]; + __align__(16) typename Traits::weight_t smem_beta[BETA_ELEMENTS]; + __align__(16) float smem_mxfp8[SMEM_MXFP8_ELEMENTS]; +}; + +extern "C" __global__ +__launch_bounds__(Ktraits::THREADS_PER_CTA, DESIRED_OCCUPANCY) void ln_fwd_kernel( + PersistentLnFwdParams params, + reduced_divisor divisor) { // divisor is div_batch if it is batch-first case, else it is div_seqLen + enum { ROWS_PER_CTA = Ktraits::ROWS_PER_CTA }; + enum { WARPS_N = Ktraits::WARPS_N }; + enum { WARPS_M = Ktraits::WARPS_M }; + enum { THREADS_PER_ROW = Ktraits::THREADS_PER_ROW }; + enum { VEC_COLS_PER_LDG = Ktraits::VEC_COLS_PER_LDG }; + enum { VEC_COLS = Ktraits::VEC_COLS }; + enum { BYTES_PER_ROW = Ktraits::BYTES_PER_ROW }; + enum { LDGS = Ktraits::LDGS }; + enum { NUM_ELTS = Ktraits::NUM_ELTS }; + enum { THREADS_PER_WARP = Ktraits::THREADS_PER_WARP }; + enum { CTAS_PER_ROW = Ktraits::CTAS_PER_ROW }; + enum { COLS = Ktraits::COLS }; + enum { COLS_PER_LDG = Ktraits::VEC_COLS_PER_LDG * Ktraits::NUM_ELTS }; + enum { COLS_PER_LDG_PER_CTA = COLS_PER_LDG / Ktraits::CTAS_PER_ROW }; + enum { VEC_COLS_PER_LDG_PER_CTA = VEC_COLS_PER_LDG / Ktraits::CTAS_PER_ROW }; + enum { USE_GAMMA_SMEM = Ktraits::USE_GAMMA_SMEM }; + enum { BATCH_SIZE = Ktraits::BATCH_SIZE }; + enum { isAdaLN = Ktraits::isAdaLN }; + enum { isBatchFirst = Ktraits::isBatchFirst }; + + using output_t = typename Ktraits::output_t; + using weight_t = typename Ktraits::weight_t; + using index_t = typename Ktraits::index_t; + using compute_t = typename Ktraits::compute_t; + using norm_output_t = typename Ktraits::norm_output_t; + using Ivec = typename Ktraits::Ivec; + using Ovec = typename Ktraits::Ovec; + using Wvec = typename Ktraits::Wvec; + using Cvec = typename Ktraits::Cvec; + using NormOvec = typename Ktraits::NormOvec; + + using Stats = typename Ktraits::Stats; + using stats_t = typename Stats::stats_t; + +#ifdef USE_STATIC_SMEM_VALUE + __shared__ __align__(16) char smem_base_[USE_STATIC_SMEM_VALUE]; +#else + extern __shared__ char smem_base_[]; +#endif + + LnFwdShared *shared = reinterpret_cast *>(smem_base_); + + uint64_t *smemBar = shared->smem_bar; +#if LN_USE_CLUSTER + if (CTAS_PER_ROW > 1) { +#if (__CUDA_ARCH__ >= 900) && (CUDART_VERSION >= 12080) + // Init the empty bars for each warp + if (threadIdx.x < WARPS_M) { + cuda::ptx::mbarrier_init(&smemBar[threadIdx.x], CTAS_PER_ROW * WARPS_N * THREADS_PER_WARP); + } + // Init the full bar (shared by the CTA) + if (threadIdx.x == 0) { + cuda::ptx::mbarrier_init(&smemBar[WARPS_M], 1); + cuda::ptx::fence_mbarrier_init(cuda::ptx::sem_release, cuda::ptx::scope_cluster); + } + cuda::ptx::barrier_cluster_arrive(cuda::ptx::sem_relaxed); + cuda::ptx::barrier_cluster_wait(); +#else + static_assert(true, "Cluster enabled on host side but not available on device"); +#endif // (__CUDA_ARCH__ >= 900) && (CUDART_VERSION >= 12080) + } +#endif // LN_USE_CLUSTER + + const index_t tidx = threadIdx.x; + const index_t bidn = blockIdx.x % CTAS_PER_ROW; + const index_t bidm = blockIdx.x / CTAS_PER_ROW; + const index_t lane = tidx % THREADS_PER_WARP; + const index_t warp = tidx / THREADS_PER_WARP; + const index_t warp_m = warp / WARPS_N; + const index_t warp_n = warp % WARPS_N; + + const index_t r = bidm * ROWS_PER_CTA + warp_m; + + const index_t col_in_tile = warp_n * THREADS_PER_WARP + lane; + const index_t c = bidn * THREADS_PER_ROW + col_in_tile; + + Stats stats(params, bidm, bidn, warp_m, warp_n, tidx, lane, shared->smem_stats, smemBar); + + // Unused when USE_GAMMA_SMEM is true and will be optimized out + [[maybe_unused]] Wvec gamma_regs[BATCH_SIZE][LDGS]; + [[maybe_unused]] Wvec beta_regs[BATCH_SIZE][LDGS]; + weight_t *gamma_smem = nullptr, *beta_smem = nullptr; + if constexpr (USE_GAMMA_SMEM) { + static constexpr int32_t SMEM_BYTES_GAMMA = THREADS_PER_ROW * BATCH_SIZE * LDGS * sizeof(Wvec); + if constexpr (Ktraits::hasGamma) { + gamma_smem = shared->smem_gamma; + } + if constexpr (Ktraits::hasBeta) { + beta_smem = shared->smem_beta; + } + } + + // If we are mxfp8 output type, we need shared memory for amax calculations across warps + // 2d1x1x (not yet implemented) requires 1 (1 for a 32x32 block) + // 1d1x1x requires 0 (since it reduces over a row which can be done with warp reduce) + // 1d2x2x requires 32x(32+1)xNUM_ELTS, more details as follows: + constexpr int block_scale_size = isFP4Out ? nvfp4_block_size : mxfp8_block_size; + BlockScaleRowHelper rowwise_scale_helper{}; + BlockScaleColHelper colwise_scale_helper{shared->smem_mxfp8}; + compute_t *mu_ptr = static_cast(params.mu); + compute_t *rs_ptr = static_cast(params.rs); + + constexpr compute_t rn = 1.f / compute_t(Ktraits::COLS); + + index_t idx = c; + +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + asm volatile("griddepcontrol.wait;\n"); +#endif + + // Load gamma and beta into shared memory or registers +#pragma unroll + for (int b = 0; b < BATCH_SIZE; b++) { +// CL-14115: The unroll factor 128 for LDGS was chosen based on the compilation/perf results for APEX LN_fwd engines +#pragma unroll 128 + for (int it = 0; it < LDGS; it++) { + if constexpr (USE_GAMMA_SMEM) { + if (warp_m == 0) { + const index_t cur_gamma_smem_base_idx = + (b * LDGS + it) * THREADS_PER_ROW * NUM_ELTS + warp_n * THREADS_PER_WARP * NUM_ELTS + lane; + if constexpr (Ktraits::hasGamma) { + Wvec cur_gamma_vec; + cur_gamma_vec.load_from(params.gamma, idx); +#pragma unroll + for (int jt = 0; jt < NUM_ELTS; jt++) { + const index_t cur_gamma_smem_idx = cur_gamma_smem_base_idx + jt * THREADS_PER_WARP; + gamma_smem[cur_gamma_smem_idx] = cur_gamma_vec.data.elt[jt]; + } + } + if constexpr (Ktraits::hasBeta) { + Wvec cur_beta_vec; + cur_beta_vec.load_from(params.beta, idx); +#pragma unroll + for (int jt = 0; jt < NUM_ELTS; jt++) { + const index_t cur_beta_smem_idx = cur_gamma_smem_base_idx + jt * THREADS_PER_WARP; + beta_smem[cur_beta_smem_idx] = cur_beta_vec.data.elt[jt]; + } + } + } + } else if constexpr (!GAMMA_ON_DEMAND) { + if constexpr (Ktraits::hasGamma) { + gamma_regs[b][it].load_from(params.gamma, idx); + } + if constexpr (Ktraits::hasBeta) { + beta_regs[b][it].load_from(params.beta, idx); + } + } + idx += VEC_COLS_PER_LDG; + } + } + + if constexpr (USE_GAMMA_SMEM) { + __syncthreads(); + } + + // Initialize scale and bias for FP8 output + compute_t scale = 1.f; + if constexpr (isFP8Out) { + scale = __ldg(params.scale); + } + compute_t amax = 0; + + index_t remaining_rows = params.rows - bidm * ROWS_PER_CTA; + int row_increment_step = params.ctas_per_col * ROWS_PER_CTA; + int batch_idx = 0, remainder = 0; + int batch_increment_step = 0, step_remainder = 0; + if constexpr (isAdaLN) { + if constexpr (isBatchFirst) { + divisor.divmod(r, batch_idx, remainder); // row = batch_idx * seqLen + remainder; (remainder < + // seqLen) batch_idx = r/seqLen + divisor.divmod(row_increment_step, batch_increment_step, step_remainder); // row_increment_step = + // batch_increment_step * seqLen + + // step_remainder (remainder < + // seqLen) + } else { + batch_idx = divisor.mod(r); // batch_idx = row % BATCH_SIZE; + batch_increment_step = + divisor.mod(row_increment_step); // batch_increment_step = row_increment_step % BATCH_SIZE; + } + } + + for (int row = r; row < params.rows; + row += row_increment_step, batch_idx += batch_increment_step, remainder += step_remainder) { + index_t idx = static_cast(row) * VEC_COLS + c; + + // Load x and convert to compute type per row per thread + compute_t xf[LDGS * NUM_ELTS]; +#pragma unroll 128 + for (int it = 0; it < LDGS; it++) { + Ivec x_it{}; + x_it.load_from(params.x, idx); +#pragma unroll + for (int jt = 0; jt < NUM_ELTS; jt++) { + compute_t x_ij = compute_t(x_it.data.elt[jt]); + xf[it * NUM_ELTS + jt] = x_ij; + } + idx += VEC_COLS_PER_LDG; + } + + // Compute mean and variance per row per thread + // How many rows current CTA will handle for this iteration + int rows_per_cta = remaining_rows >= ROWS_PER_CTA ? ROWS_PER_CTA : remaining_rows; + stats_t s = stats.compute(xf, rn, rows_per_cta); + remaining_rows -= params.ctas_per_col * ROWS_PER_CTA; // for next iteration + compute_t mu = Get<0>::of(s); + compute_t m2 = Get<1>::of(s); + if constexpr (!Ktraits::isRMSNorm) { + if (bidn == 0 && warp_n == 0 && lane == 0) { + mu_ptr[row] = mu; + } + } + compute_t rs = rsqrtf(rn * m2 + params.epsilon); + + if (bidn == 0 && warp_n == 0 && lane == 0) { + rs_ptr[row] = rs; + } + + idx = row * VEC_COLS + c; + + if constexpr (isAdaLN) { + if constexpr (isBatchFirst) { + if (remainder >= params.seqLen) { + batch_idx += 1; + remainder -= params.seqLen; + } + } else { + if (batch_idx >= BATCH_SIZE) { + batch_idx -= BATCH_SIZE; + } + } + } + index_t gamma_idx = c + (batch_idx * LDGS) * VEC_COLS_PER_LDG; +#pragma unroll 128 + for (int it = 0; it < LDGS; it++) { + Cvec z_math; + [[maybe_unused]] Wvec g_wt; + [[maybe_unused]] Wvec b_wt; + if constexpr (GAMMA_ON_DEMAND && Ktraits::hasGamma && !USE_GAMMA_SMEM) { + g_wt.load_from(params.gamma, gamma_idx); + + if constexpr (Ktraits::hasBeta) { + b_wt.load_from(params.beta, gamma_idx); + } + } +#pragma unroll + // Compute output per ldg per row per thread + for (int jt = 0; jt < NUM_ELTS; jt++) { + compute_t y_ij = rs * (xf[it * NUM_ELTS + jt] - mu); + + if constexpr (Ktraits::hasGamma) { + weight_t g_ij_wt{}; + const int32_t cur_gamma_smem_base_idx = (batch_idx * LDGS + it) * THREADS_PER_ROW * NUM_ELTS + + warp_n * THREADS_PER_WARP * NUM_ELTS + lane; + const int32_t cur_gamma_smem_idx = cur_gamma_smem_base_idx + jt * THREADS_PER_WARP; + if constexpr (USE_GAMMA_SMEM) { + g_ij_wt = gamma_smem[cur_gamma_smem_idx]; + } else if constexpr (GAMMA_ON_DEMAND) { + g_ij_wt = g_wt.data.elt[jt]; + } else { + g_ij_wt = gamma_regs[batch_idx][it].data.elt[jt]; + } + compute_t g_ij = static_cast(g_ij_wt); + if constexpr (isZeroCenteredGamma) { + if constexpr (isZeroCenteredGammaCastBeforeAdd) { + g_ij = static_cast(g_ij_wt) + static_cast(1.f); + } else { + g_ij = static_cast(g_ij_wt + static_cast(1.f)); + } + } + if constexpr (Ktraits::hasBeta) { + compute_t b_ij{}; + const int32_t cur_beta_smem_idx = cur_gamma_smem_base_idx + jt * THREADS_PER_WARP; + if constexpr (USE_GAMMA_SMEM) { + b_ij = beta_smem[cur_beta_smem_idx];)KERNEL" + R"KERNEL( } else if constexpr (GAMMA_ON_DEMAND) { + b_ij = static_cast(b_wt.data.elt[jt]); + } else { + b_ij = beta_regs[batch_idx][it].data.elt[jt]; + } + y_ij = g_ij * y_ij + b_ij; + } else { + y_ij = g_ij * y_ij; + } + } + + // SiLU activation: y = y * sigmoid(y) = y / (1 + exp(-y)) + // Applied after norm + gamma [+ beta], before FP8/block-scale quantization. + y_ij = __fdividef(y_ij, 1.0f + __expf(-y_ij)); + + if constexpr (isFP8Out) { + if (hasAmax) { + __builtin_assume(amax >= 0); + amax = fmaxf(amax, fabsf(y_ij)); + } + y_ij *= scale; + } + z_math.data.elt[jt] = y_ij; + } // NUM_ELTS + + if constexpr (isBlockScaleOut) { + static_assert(!isBlockScaleOut || + (Ktraits::COLS % mxfp8_block_size == 0)); // ensure cols divisable by 32 + + index_t sf_row_idx = idx / mxfp8_block_size; + [[maybe_unused]] NormOvec z_intermediate; + if constexpr (std::is_same::value) { + rowwise_scale_helper.blockQuantizeStore(z_math, params.scale_row, sf_row_idx, params.z, idx); + } else { + z_math.to(z_intermediate); + rowwise_scale_helper.blockQuantizeStore( + z_intermediate, params.scale_row, sf_row_idx, params.z, idx); + } + if constexpr (isBlockScale_1D2X2X) { + if constexpr (useBlockScaleColwiseKernel) { + // Store the temporary z_math values in workspace and launch a separate kernel to compute the + // colwise scaling results + if constexpr (std::is_same::value) { + z_math.store_to(params.z_math, idx); + } else { + z_intermediate.store_to(params.z_math, idx); + } + } else { + if constexpr (std::is_same::value) { + colwise_scale_helper.initTile(z_math, THREADS_PER_ROW * WARPS_M); + // static_assert(!std::is_same::value); + } else { + colwise_scale_helper.initTile(z_intermediate, THREADS_PER_ROW * WARPS_M); + } + index_t sf_col_row_idx = 0; + index_t sf_col_col_idx = 0; + index_t sf_col_row_width = 0; + index_t z_col_idx = 0; + index_t z_row_offset = row - row % mxfp8_block_size; + if constexpr (!isBlockScale_1D2X2X_Transpose) { + sf_col_row_idx = row / mxfp8_block_size; + sf_col_col_idx = it * VEC_COLS_PER_LDG + bidn * VEC_COLS_PER_LDG_PER_CTA + warp; + sf_col_row_width = VEC_COLS; + z_col_idx = (z_row_offset + lane) * VEC_COLS + sf_col_col_idx; + } else { + constexpr index_t group_size = mxfp8_block_size / NUM_ELTS; + sf_col_row_idx = it * COLS_PER_LDG + bidn * COLS_PER_LDG_PER_CTA + warp * NUM_ELTS; + sf_col_col_idx = row / mxfp8_block_size; + sf_col_row_width = params.rows / mxfp8_block_size; + z_col_idx = (sf_col_row_idx + lane / group_size) * params.rows / NUM_ELTS + + z_row_offset / NUM_ELTS + (lane % group_size); + } + colwise_scale_helper.blockQuantizeStore( + params.scale_col, + sf_col_row_idx, + sf_col_col_idx, + sf_col_row_width, + params.z_col, + z_col_idx, + THREADS_PER_ROW * WARPS_M); + } + } + } else { + Ovec z; + z_math.to(z); + z.store_to(params.z, idx); + } + idx += VEC_COLS_PER_LDG; + gamma_idx += VEC_COLS_PER_LDG; + } // LDGS + } // grid stride loop + + // Write scale_inv before launch_dependents - consumer needs it to dequantize FP8 output + if constexpr (isFP8Out) { + if (hasScaleInv && blockIdx.x == 0 && threadIdx.x == 0) { + *reinterpret_cast(params.scale_inv) = __fdividef(1.f, scale); + } + } + +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + asm volatile("griddepcontrol.launch_dependents;\n"); +#endif + + // amax can be after launch_dependents - only needed for delayed scaling (next iteration) + if constexpr (isFP8Out) { + if constexpr (hasAmax) { + amax = reduce_max(amax, warp, threadIdx.x); + if (threadIdx.x == 0) { + atomicMaxFloat(reinterpret_cast(params.amax), amax); + } + } + } +} + +)KERNEL"; +inline constexpr size_t ln_fwd_silu_kernel_source_len = sizeof(ln_fwd_silu_kernel_source) - 1; + +} // namespace cudnn_frontend::experimental::generated + +#if defined(_MSC_VER) +#pragma warning(pop) +#endif + +#if defined(__clang__) +#pragma clang diagnostic pop +#endif + +namespace cudnn_frontend::experimental::generated { + +inline constexpr const char ln_fwd_silu_kernel_flags[] = R"FLAGS(--gpu-architecture=sm_100a +--std=c++17 +-w +--define-macro=__CUDACC_RTC__ +-default-device +--use_fast_math +)FLAGS"; +inline constexpr size_t ln_fwd_silu_kernel_flags_len = sizeof(ln_fwd_silu_kernel_flags) - 1; + +} // namespace cudnn_frontend::experimental::generated diff --git a/deps/cudnn-frontend/include/cudnn_frontend/generated/rms_norm_silu/sm100/ln_headers.h b/deps/cudnn-frontend/include/cudnn_frontend/generated/rms_norm_silu/sm100/ln_headers.h new file mode 100644 index 0000000..59bac63 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/generated/rms_norm_silu/sm100/ln_headers.h @@ -0,0 +1,2043 @@ +// Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: MIT +#pragma once +// Auto-generated — do not edit manually. +// Lightweight LN headers (~3K lines) replacing the 151K-line embedded version. +// Uses #include directives for CUDA TK headers (requires --include-path at NVRTC compile time). + +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Woverlength-strings" +#endif + +#if defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable : 4068) +#endif + +namespace cudnn_frontend { +namespace experimental { +namespace generated { + +inline constexpr const char ln_headers_source[] = + R"KERNEL( +#pragma once +// Auto-generated lightweight LN kernel header. +// Replaces the 151K-line persistent_ln_headers_13.0.h with: +// - Standard CUDA TK #include directives (resolved via --include-path at NVRTC compile time) +// - ~2900 lines of cuDNN-authored LN-specific code extracted from the original +// +// CGA/cluster support (USE_CLUSTER) is disabled: +// - #include and cooperative_groups are NOT included +// - All if constexpr (USE_CLUSTER) branches changed to if constexpr(USE_CLUSTER) so dead code is not compiled +// +// Requires CUDA Toolkit headers available at the --include-path location. +// Generated by step3b_extract_ln_headers.py — do not edit manually. + +// ============================================================ +// Standard CUDA Toolkit headers (resolved via --include-path) +// ============================================================ + +// Minimal std stubs needed by NVRTC (no standard library available) +#if __cplusplus >= 201103L +namespace std { +template struct enable_if {}; +template struct enable_if { typedef T type; }; +#if __cplusplus >= 201402L +template< bool B, class T = void > using enable_if_t = typename enable_if::type; +#endif + +struct true_type { enum { value = true }; operator bool() const { return true; } }; +struct false_type { enum { value = false }; operator bool() const { return false; } }; + +template struct is_floating_point : false_type {}; +template<> struct is_floating_point : true_type {}; +template<> struct is_floating_point : true_type {}; + +template struct is_integral : false_type {}; +template<> struct is_integral : true_type {}; +template<> struct is_integral : true_type {}; +template<> struct is_integral : true_type {}; +template<> struct is_integral : true_type {}; +template<> struct is_integral : true_type {}; +template<> struct is_integral : true_type {}; +template<> struct is_integral : true_type {}; +template<> struct is_integral : true_type {}; +template<> struct is_integral : true_type {}; +template<> struct is_integral : true_type {}; +template<> struct is_integral : true_type {}; + +template struct is_same : false_type {}; +template struct is_same : true_type {}; +} +#endif + +// CUDA numeric type headers +#include +#include +#include +#include + +// NOTE: #include and are intentionally omitted. +// They are only needed for CGA/cluster support (USE_CLUSTER=true), which this +// OSS engine does not use. All USE_CLUSTER code paths are guarded with +// "if constexpr" so dead branches are not compiled. + +// ============================================================ +// Fixed-width integer typedefs for NVRTC (no available) +// ============================================================ +constexpr int THREADS_PER_WARP = 32; + +#ifndef __CUDACC_RTC__ +#include +#else +typedef signed char int8_t; +typedef unsigned char uint8_t; +typedef signed short int16_t; +typedef unsigned short uint16_t; +typedef signed int int32_t; +typedef unsigned int uint32_t; +typedef signed long long int64_t; +typedef unsigned long long uint64_t; +#endif + +// ============================================================ +// cuDNN-authored LN kernel utilities +// Extracted from persistent_ln_headers_13.0.h +// ============================================================ + + +// cuDNN type aliases for FP8 types (from after inlined cuda_fp8.h) +typedef __nv_fp8_e4m3 nv_fp8_e4m3; +typedef __nv_fp8x2_e4m3 nv_fp8x2_e4m3; +typedef __nv_fp8_e5m2 nv_fp8_e5m2; +typedef __nv_fp8x2_e5m2 nv_fp8x2_e5m2; + +// cuDNN type aliases for FP4 types (from after inlined cuda_fp4.h) +typedef __nv_fp4_e2m1 nv_fp4_e2m1; +typedef __nv_fp4x2_e2m1 nv_fp4x2_e2m1; +typedef __nv_fp4x4_e2m1 nv_fp4x4_e2m1; + +// Pointer typedefs for NVRTC (from original Jitify stubs) +typedef int64_t intptr_t; +typedef uint64_t uintptr_t; +typedef int64_t intmax_t; +typedef uint64_t uintmax_t; + +// Large vector types for 64-byte and 32-byte loads +struct uint16 { + uint4 u; + uint4 v; + uint4 s; + uint4 t; +}; + +struct uint8 { + uint4 u; + uint4 v; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// +//// Datatype helpers + +template +struct BytesToType {}; + +template <> +struct BytesToType<64> { + using Type = uint16; +}; + +template <> +struct BytesToType<32> { + using Type = uint8; +}; + +template <> +struct BytesToType<16> { + using Type = uint4; +}; + +template <> +struct BytesToType<8> { + using Type = uint64_t; +}; + +template <> +struct BytesToType<4> { + using Type = uint32_t; +}; + +template <> +struct BytesToType<2> { + using Type = uint16_t; +}; + +template <> +struct BytesToType<1> { + using Type = uint8_t; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct TypeToVec2 {}; + +template <> +struct TypeToVec2 { + using Type = float2; +}; + +template <> +struct TypeToVec2 { + using Type = half2; +}; + +template <> +struct TypeToVec2 { + using Type = nv_bfloat162; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct Get { + template + static inline __device__ R + of(const T& vec); +}; + +template <> +template +inline __device__ R +Get<0>::of(const T& vec) { + return vec.x; +} + +template <> +template +inline __device__ R +Get<1>::of(const T& vec) { + return vec.y; +} + +template <> +template +inline __device__ R +Get<2>::of(const T& vec) { + return vec.z; +} + +template <> +template +inline __device__ R +Get<3>::of(const T& vec) { + return vec.w; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct Converter { + static inline __device__ Dst + convert(const Src& from) { + return Dst(from); + } +}; + +template <> +struct Converter { + static inline __device__ half2 + convert(const float2& x) { + return __float22half2_rn(x); + } +}; + +template <> +struct Converter { + static inline __device__ nv_bfloat162 + convert(const float2& x) { +#if __CUDA_ARCH__ >= 800 + return __float22bfloat162_rn(x); +#else + union { + nv_bfloat162 raw; + nv_bfloat16 x; + nv_bfloat16 y; + } tmp; + tmp.x = __float2bfloat16_rn(x.x); + tmp.y = __float2bfloat16_rn(x.y); + return tmp.raw; +#endif + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct Zeros { + static inline __device__ T + get() { + return T(0.f); + } +}; + +template <> +struct Zeros { + static inline __device__ float2 + get() { + return make_float2(0.f, 0.f); + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +__device__ __inline__ uint8_t +float_to_e8m0(float val) { + // CL-15277: use rounding-up mode for float -> e8m0 conversion +#if (defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 1000) && \ + (defined(__CUDA_ARCH_FEAT_SM100_ALL) || defined(__CUDA_ARCH_FEAT_SM101_ALL) || defined(__CUDA_ARCH_FEAT_SM110_ALL) || \ + defined(__CUDA_ARCH_FEAT_SM120_ALL)) + uint16_t out; + asm volatile( + "{\n" + "cvt.rp.satfinite.ue8m0x2.f32 %0, 0.0, %1;\n" + "}" + : "=h"(out) + : "f"(val)); + return *reinterpret_cast(&out); +#else +#if CUDART_VERSION >= 12080 + // Use explicit round-up mode to match the SM_100+ hardware instruction + return __nv_cvt_float_to_e8m0(val, __NV_SATFINITE, cudaRoundPosInf); +#else + if (isnan(val) || isinf(val)) { + return 0xFF; + } + uint32_t val_u32 = __float_as_uint(val); + uint8_t exponent = (val_u32 >> 23) & 0xFF; + uint32_t mantissa = val_u32 & 0x7FFFFF; + if ((mantissa > 0) && (exponent != 0xFE)) { // exp can only be < 0xFE here + exponent++; + } + return exponent; +#endif +#endif +} + +__device__ __inline__ float +e8m0_to_float(uint8_t val) { +#if CUDART_VERSION >= 12080 + __nv_fp8_e8m0 e8m0; + e8m0.__x = val; + return static_cast(e8m0); +#else + return __uint_as_float(static_cast(val) << 23); +#endif + +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// +//// Vectorization data structures +template +struct Vec { + using Elt_type = ELT_TYPE; + + enum { BYTES = NUM_ELT * sizeof(Elt_type) }; + using Vec_type = typename BytesToType::Type; + + using Alias_type = union { + Vec_type vec; + Elt_type elt[NUM_ELT]; + }; + + Alias_type data; + + template + inline __device__ void + to(Vec& other) { +#pragma unroll + for (int it = 0; it < NUM_ELT; it++) { + other.data.elt[it] = S(this->data.elt[it]); + } + } + + template + inline __device__ void + assign(const Op& op) { +#pragma unroll + for (int it = 0; it < NUM_ELT; it++) { + this->data.elt[it] = op(it); + } + } + + inline __device__ void + load_from(const void* base_ptr, const size_t idx) { + this->data.vec = static_cast(base_ptr)[idx]; + } + + inline __device__ void + store_to(void* base_ptr, const size_t idx) { + static_cast(base_ptr)[idx] = this->data.vec; + } + + inline __device__ void + operator+=(const Vec& rhs) { +#pragma unroll + for (int it = 0; it < NUM_ELT; it++) { + this->data.elt[it] += rhs.data.elt[it]; + } + } +}; + +inline __device__ float2 +operator+(const float2& a, const float2& b) { + return {a.x + b.x, a.y + b.y}; +} + +inline __device__ void +operator+=(float2& a, const float2& b) { + a.x += b.x; + a.y += b.y; +} + +template +struct Sum { + inline __device__ + Sum() {} + inline __device__ T + operator()(const T& a, const T& b) { + return a + b; + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +inline __device__ void +__nv_ptx_builtin_ocg_fence_view_async_shared(void) { + asm volatile("fence.proxy.async.shared::cta;\n"); +} + +extern "C" { +__device__ uint32_t +__nvvm_get_smem_pointer(void* ptr); +//__device__ void +//__nv_ptx_builtin_ocg_fence_view_async_shared(void); +__device__ void +__nv_ptx_builtin_ocg_write_async_shared_b32(uint32_t dstAddr, uint32_t mbarrierAddr, uint32_t b0); +__device__ void +__nv_ptx_builtin_ocg_write_async_shared_b64(uint32_t dstAddr, uint32_t mbarrierAddr, uint64_t b0); +__device__ void +__nv_ptx_builtin_ocg_write_async_shared_v2_b32(uint32_t dstAddr, uint32_t mbarrierAddr, uint32_t b0, uint32_t b1); +__device__ void +__nv_ptx_builtin_ocg_write_async_shared_v4_b32(uint32_t dstAddr, + uint32_t mbarrierAddr, + uint32_t b0, + uint32_t b1, + uint32_t b2, + uint32_t b3); +} + +namespace utils { + +////////////////////////////////////////////////////////////////////////////////////////////////// +//// Shuffle syncronization helpers +template +inline __device__ T +warp_shuffle_xor(const T& x, uint32_t idx) { + return __shfl_xor_sync(uint32_t(-1), x, idx); +} + +template <> +inline __device__ float2 +warp_shuffle_xor(const float2& x, uint32_t idx) { + return {warp_shuffle_xor(x.x, idx), warp_shuffle_xor(x.y, idx)};)KERNEL" + R"KERNEL(} + +template +inline __device__ T +warp_shuffle_down(const T& x, uint32_t idx) { + return __shfl_down_sync(uint32_t(-1), x, idx); +} + +template <> +inline __device__ float2 +warp_shuffle_down(const float2& x, uint32_t idx) { + return {warp_shuffle_down(x.x, idx), warp_shuffle_down(x.y, idx)}; +} + +// This is a helper function to support __shfl_sync(float2) +template +__device__ inline static T +shfl_sync_helper(unsigned mask, T var, int srcLane) { + return __shfl_sync(mask, var, srcLane); +} + +// specialize for float2 +template <> +__device__ inline float2 +shfl_sync_helper(unsigned mask, float2 var, int srcLane) { + double ret = __shfl_sync(mask, *reinterpret_cast(&var), srcLane); // cast to double + return *reinterpret_cast(&ret); +} + +////////////////////////////////////////////////////////////////////////////////////////////////// +__device__ __forceinline__ void +namedBarrierSync(int name, int numThreads) { + asm volatile("bar.sync %0, %1;" : : "r"(name), "r"(numThreads) : "memory"); +} + +} // namespace utils + +//////////////////////////////////////////////////////////////////////////////////////////////////// +//// Atomics +static inline __device__ void +atomicMaxFloat(float* addr, const float value) { + atomicMax(reinterpret_cast(addr), __float_as_int(value)); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// +//// Warp reduction functions +template +static inline __device__ float +warp_reduce_max(const float m, unsigned int mask = 0xFFFFFFFF) { + static constexpr int WARP_SIZE = 32; + static_assert(GROUP_SIZE > 0 && GROUP_SIZE <= WARP_SIZE, "group size must be less than or equal to warp size"); + + float max_val = m; +#if (defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 1000) && defined(__CUDA_ARCH_FEAT_SM100_ALL) + // Only enable the credux instruction for reduction on the full warp, as we saw perf regression for GROUP_SIZE < 32, + // where we need to iterate over #(WARP_SIZE/GROUP_SIZE) groups in the warp + if constexpr (GROUP_SIZE == WARP_SIZE) { + // For sm_100a arch, we can use redux.sync.op{.abs.}{.NaN}.f32 instruction to reduce the min/max value + // https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-redux-sync + asm volatile("redux.sync.max.f32 %0, %1, %2;" : "=f"(max_val) : "f"(m), "r"(mask)); + return max_val; + } +#endif +#pragma unroll + for (int delta = GROUP_SIZE / 2; delta > 0; delta /= 2) { + const float other_m = __shfl_down_sync(mask, max_val, delta); + __builtin_assume(max_val >= 0); + __builtin_assume(other_m >= 0); + max_val = fmaxf(max_val, other_m); + } + return max_val; +} + +template +static inline __device__ compute_t +reduce_max(const compute_t m, const int warpid) { + __shared__ float staging[num_warps]; + constexpr int warp_size = 32; + const float my_max = m; + const float my_warp_max = warp_reduce_max(my_max); + if (threadIdx.x % 32 == 0) { + staging[warpid] = my_warp_max; + } + __syncthreads(); + compute_t result = 0; + if (warpid == 0) { + const float my_max = threadIdx.x < num_warps ? staging[threadIdx.x] : 0; + result = warp_reduce_max(my_max); + } + return result; +} + +template +static inline __device__ compute_t +reduce_max(const compute_t m, const int warpid, int tidx) { + __shared__ float staging[num_warps]; + constexpr int warp_size = 32; + const float my_max = m; + const float my_warp_max = warp_reduce_max(my_max); + if (tidx % 32 == 0) { + staging[warpid] = my_warp_max; + } + utils::namedBarrierSync(1, num_warps * warp_size); + compute_t result = 0; + if (warpid == 0) { + const float my_max = tidx < num_warps ? staging[tidx] : 0; + result = warp_reduce_max(my_max); + } + return result; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// +//// Misc +constexpr __device__ int32_t +find_largest_divisor(int32_t n, int32_t divisor_upper_bound) { + if (n <= divisor_upper_bound) { + return n; + } + for (int32_t i = divisor_upper_bound; i >= 1; --i) { + if (n % i == 0) { + return i; // found the largest divisor + } + } + return divisor_upper_bound; // fallback +} + +constexpr __device__ bool +is_power_of_2(unsigned int n) { + return (n > 0) && ((n & (n - 1)) == 0); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// +//// Cooperative groups related +//// !!!! Keep this block at the end of the file - start !!!! + +//////////////////////////////////////////////////////////////////////////////////////////////////// +//// Runtime parameters + +struct PersistentLnParamsBase { + // For Multi-CTA, number of different CTA groups; otherwise same as gridDim.x. + int ctas_per_col = 0; + + // Input is interpreted as matrix and we normalize across columns. + int rows = 0; + int cols = 0; + int batchSize = 1; // For AdaLN + int seqLen = 1; // For AdaLN + + // Common data pointers for forward and backward passes. + void *x = nullptr; + void *mu = nullptr; + void *rs = nullptr; + void *gamma = nullptr; + + // Multi-CTA workspace in gmem. + void *workspace = nullptr; + + // Multi-CTA sync barriers in gmem. + int *barrier = nullptr; + + bool isRMSNorm = false; + bool noScale = false; + bool noBias = false; + bool isAdaLN = false; + bool isBatchFirst = true; +}; + +struct PersistentLnFwdParams : public PersistentLnParamsBase { + // Output of LN FWD. + void *z = nullptr; + + void *beta = nullptr; + float epsilon = 0.f; + + // FP8 support + bool fp8_out = false; + float *scale = nullptr; + float *scale_inv = nullptr; + float *amax = nullptr; + + // Reuse z for row scaled output (or block scaled output if we use 2d1x1x) + void *scale_row = nullptr; + void *scale_col = nullptr; // only used in 1d2x2x + void *z_col = nullptr; // only used in 1d2x2x + void *z_math = nullptr; // only used when enabling the 1d2x2x colwise kernel +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// +//// Inter-CTA synchronization +template +struct InterCTASync { + template + inline __device__ + InterCTASync(Params ¶ms, uint32_t bidm, uint32_t bidn, uint32_t tidx) + : phase_counter_(0), + b0_(params.barrier + bidm) // The barrier for this group of CTAs. + , + b1_(params.barrier + bidm + params.ctas_per_col) // The barrier for this group of CTAs. + , + tidx_(tidx) { + // BARRIERS ARE ASSUMED TO BE INITIALIZED TO 0! + } + + inline __device__ void + spin_wait_(int *barrier, int step, int expected) { + asm volatile("red.release.gpu.global.add.s32 [%0], %1;" ::"l"(barrier), "r"(step)); + for (int found = -1; found != expected;) { + asm volatile("ld.global.acquire.gpu.b32 %0, [%1];" : "=r"(found) : "l"(barrier)); + } + } + + inline __device__ void + sync(uint32_t threads_per_cta) { + // ALL THREADS MUST ENTER! + + // We switch barrier every iteration. + int *barrier = phase_counter_ & 0x1 ? b1_ : b0_; + // We decrement every other iteration. + bool dec = phase_counter_ & 0x2; + int step = dec ? -1 : 1; + int expected = dec ? 0 : CTAS_PER_ROW; + // There are only 4 phases: up/down for b0/b1. + phase_counter_ = (phase_counter_ + 1) & 0x3; + + if (0 == tidx_) { + spin_wait_(barrier, step, expected); + } + + // CTA waits for thread 0 + utils::namedBarrierSync(0, threads_per_cta); + } + + int phase_counter_; + int *b0_; + int *b1_; + uint32_t tidx_; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// +//// LayerNorm Reduction and stats computation +// NOTE: WHOLE_CTA means if we'll use one empty mbarrier for inter-CTA sync using cluster. +// In the TMA LN engines it should be "true" since we're syncing the whole CTA with one empty mbarrier, while in the +// APEX LN engines it should be "false" as there's one empty mbarrier per warp m (and therefore WARPS_M empty mbarriers +// in total) +template +struct Reducer : public Reducer { + using InterCTASync = InterCTASync; + using Base = Reducer; + using Type = typename Base::Type; + + enum { SMEM_BYTES = Base::SMEM_BYTES }; + + enum { WS_BARRIER_BYTES = 2 * sizeof(int) }; + enum { WS_DATA_BYTES = WARPS_M * CTAS_PER_ROW * sizeof(T) }; + + // size of the barriers + temporary result per CTA (multiply with CTAS_PER_ROW to get total) + enum { WORKSPACE_BYTES_PER_GROUP = Base::WORKSPACE_BYTES_PER_GROUP + WS_BARRIER_BYTES + WS_DATA_BYTES }; + + template + inline __device__ + Reducer(Params ¶ms, + uint32_t bidm, + uint32_t bidn, + uint32_t warp_m, + uint32_t warp_n, + uint32_t tidx, + uint32_t lane, + void *smem, + uint64_t *smem_bar) + : Base(params, bidm, bidn, warp_m, warp_n, tidx, lane, smem, smem_bar), + inter_cta_(params, bidm, bidn, tidx), + bidn_(bidn), // CTA id within the group. + warp_m_(warp_m), + w0_(static_cast(params.workspace) + (bidm * WARPS_M + warp_m) * CTAS_PER_ROW), + w1_(w0_ + params.ctas_per_col * WARPS_M * CTAS_PER_ROW), + smem_bar_(smem_bar), + parity_(0) { + // the first several elements are the barriers + smem_cga_ = reinterpret_cast(&smem_bar[WHOLE_CTA? 2: (WARPS_M + 1)]); + } + + template + inline __device__ T + allreduce(T data, Op &op, int32_t warps_m) { + data = Base::reduce(data, op, warps_m); + + if constexpr (USE_CLUSTER) { +#if (__CUDA_ARCH__ >= 900) && (CUDART_VERSION >= 12080) + auto cluster = cooperative_groups::this_cluster(); + + // NOTE: right now only lane 0 has the valid value so that a intra-warp broadcast is required here. + // This issue is not in fwd (struct Stats) which does it earlier inside warp_chan_upd_dynamic(). + data = utils::shfl_sync_helper(uint32_t(-1), data, 0); + + // Broadcast local results to other CTAs inside the CGA + // Size of smem_cga_: [WARPS_M][CTAS_PER_ROW] + // Thread 0 sends to block 0, Thread 1 sends to block 1, etc... + if ((this->warp_n_ == 0) && (this->lane_ < CTAS_PER_ROW)) { + st_async_remote(&smem_cga_[warp_m_ * CTAS_PER_ROW + bidn_], data, this->lane_, &smem_bar_[WHOLE_CTA? 0: WARPS_M]); + } + + // Leader thread arrives on local barrier to indicate expected tx count + // We can't use (threadIdx.x == 0) directly here because it won't work for TMA + if ((this->warp_n_ == 0) && (warp_m_ == 0) && (this->lane_ == 0)) { + // It's possible that not all warp_m are active here, therefore, we can't use WARPS_M directly + uint32_t expected_tx_count = sizeof(data) * CTAS_PER_ROW * warps_m; + utils::mbarrier_arrive_expect_tx_relaxed_cluster(&smem_bar_[WHOLE_CTA? 0: WARPS_M], expected_tx_count); + } + + // Wait on local barrier + while (!utils::mbarrier_try_wait_parity_relaxed_cluster(&smem_bar_[WHOLE_CTA? 0: WARPS_M], parity_)) {} + utils::fence_acquire_smem(); // Ensure we can read local smem values that are released by barrier flip + + T total = Zeros::get(); + if (this->lane_ < CTAS_PER_ROW) { + total = smem_cga_[warp_m_ * CTAS_PER_ROW + this->lane_]; + } + utils::fence_release_smem(); // Ensure read of buffer remains ordered before smem_bar.arrive.relaxed +)KERNEL" + R"KERNEL( // Signal barrier is empty. + // Each thread must arrive on all the barriers (like an arrive broadcast). + // NOTE: here we only let each warp m arrive on the remote barriers of the same warp m (i.e. the same row), + // because if we have one empty barrier for the entire CTA, there's a difficulty how to init the count of + // the barrier (since some warps could be disabled here). The downside is we'll need a __syncthreads() here, + // which doesn't have obvious perf impact though. + // TODO: we could probably have one full bar per row (same as the empty bar) to get rid of the CTA sync, + // while the downside is more smem (mbarriers) will be needed. + if (!WHOLE_CTA) { + utils::namedBarrierSync(0, warps_m * WARPS_N * 32); + } + for (int other_block_rank = 0; other_block_rank < CTAS_PER_ROW; ++other_block_rank) { + uint64_t *remote_bar_empty = cluster.map_shared_rank(&smem_bar_[WHOLE_CTA? 1: warp_m_], other_block_rank); + utils::mbarrier_arrive_relaxed_cluster(remote_bar_empty); + } + + total = Reducer::allreduce_(total, op); + + // Wait for remote buffer to be writable again by waiting on local barrier: + while (!utils::mbarrier_try_wait_parity_relaxed_cluster(&smem_bar_[WHOLE_CTA? 1: warp_m_], parity_)) {} + utils::fence_acquire_smem(); // Order wrt st_async_remote + + // Flip parity bit for next iteration + parity_ ^= 1; + + return total; +#else + static_assert(true, "Cluster enabled on host side but not available on device"); +#endif + } else { + // We switch workspace every iteration. + T *workspace = inter_cta_.phase_counter_ & 0x1 ? w1_ : w0_; + + // Warp leaders 0 hold the CTA-local results. + if (this->warp_n_ == 0 && this->lane_ == 0) { + workspace[bidn_] = data; + } + + // We need to check for the case where we are using CTA > 1 and WARP_M > 1 and N == 1 + // Because in this case we cannot rely on the __syncThreads in template::compute (because we will use the specialized template ::compute which does not have a sync) + if (CTAS_PER_ROW > 1 && WARPS_M > 1 && WARPS_N == 1) { + utils::namedBarrierSync(0, warps_m * WARPS_N * 32); + } + + // Wait for all CTAS_PER_ROW CTAS in the group to have written their result. + inter_cta_.sync(warps_m * WARPS_N * 32); + + T total = Zeros::get(); + + // Every warp does the final reduction locally. + if constexpr (CTAS_PER_ROW <= THREADS_PER_WARP) { + if (this->lane_ < CTAS_PER_ROW) { + total = workspace[this->lane_]; + } + } else { + static constexpr int32_t LOOP_PER_THREAD = (CTAS_PER_ROW + THREADS_PER_WARP - 1) / THREADS_PER_WARP; + + // Collect stats for the current lane from ctas (#bidn % THREADS_PER_WARP == #lane) and then do the + // intra-warp reduction + total = workspace[this->lane_]; +#pragma unroll + for (int32_t loop_in_thread = 1; loop_in_thread < LOOP_PER_THREAD; ++loop_in_thread) { + const int32_t lane_in_loop = this->lane_ + loop_in_thread * THREADS_PER_WARP; + if (lane_in_loop < CTAS_PER_ROW) { + total = op(total, workspace[lane_in_loop]); + } + } + } + + total = Reducer::allreduce_(total, op); + + return total; + } + } + + InterCTASync inter_cta_; + T *w0_; + T *w1_; + int bidn_; + int warp_m_; + + private: + T *smem_cga_; + uint64_t *smem_bar_; + int parity_; +}; + +template +struct Reducer { + using Type = T; + enum { SMEM_BYTES = 0 }; + enum { WORKSPACE_BYTES_PER_GROUP = 0 }; + enum { THREADS_PER_WARP = 32 }; + + template + inline __device__ + Reducer(Params ¶ms, + uint32_t bidm, + uint32_t bidn, + uint32_t warp_m, + uint32_t warp_n, + uint32_t /*tidx*/, + uint32_t lane, + void *smem, + uint64_t *smem_bar) + : warp_n_(warp_n), lane_(lane) {} + + template + static inline __device__ T + allreduce_(T data, Op &op) { +#pragma unroll + for (int it = 1; it < THREADS_PER_WARP; it *= 2) { + data = op(data, utils::warp_shuffle_xor(data, it)); + } + return data; + } + + template + inline __device__ T + allreduce(T data, Op &op, int32_t warps_m) { + return allreduce_(data, op); + } + + protected: + template + inline __device__ T + reduce(T data, Op &op, int32_t warps_m) { +// only lane 0 holds the result! +#pragma unroll + for (int it = THREADS_PER_WARP / 2; it > 0; it /= 2) { + data = op(data, utils::warp_shuffle_down(data, it)); + } + return data; + } + + public: + int warp_n_; + int lane_; +}; + +template +struct Reducer : public Reducer { + using Base = Reducer; + using Type = T; + + enum { SMEM_BYTES = Base::SMEM_BYTES + WARPS_M * WARPS_N * sizeof(T) * 2 }; + enum { WORKSPACE_BYTES_PER_GROUP = 0 }; + enum { THREADS_PER_WARP = 32 }; + + template + inline __device__ + Reducer(Params ¶ms, + uint32_t bidm, + uint32_t bidn, + uint32_t warp_m, + uint32_t warp_n, + uint32_t tidx, + uint32_t lane, + void *smem, + uint64_t *smem_bar) + : Base(params, bidm, bidn, warp_m, warp_n, tidx, lane, smem, smem_bar), use0_(true) { + smem0_ = &static_cast(smem)[warp_m * WARPS_N]; + smem1_ = smem0_ + WARPS_M * WARPS_N; + } + + template + inline __device__ T + allreduce(T data, Op &op, int32_t warps_m) { + T *smem = use0_ ? smem0_ : smem1_; + use0_ = !use0_; + data = Base::reduce(data, op, warps_m); + if (this->lane_ == 0) { + smem[this->warp_n_] = data; + } + // The number matters here. 0 will only use one barrier per block. "1" will use 2 according to ncu. + // We should use 0 unless it is absolutely necessary to use seperate barriers. + utils::namedBarrierSync(0, warps_m * WARPS_N * 32); + T out = Zeros::get(); +#pragma unroll + for (int it = 0; it < WARPS_N; it++) { + out = op(out, smem[it]); + } + return out; + } + + protected: + template + inline __device__ T + reduce(T data, Op &op, int32_t warps_m) { + T *smem = use0_ ? smem0_ : smem1_; + use0_ = !use0_; + // only intra-CTA group leader holds the result! + data = Base::reduce(data, op, warps_m); + if (this->lane_ == 0) { + smem[this->warp_n_] = data; + } + utils::namedBarrierSync(0, warps_m * WARPS_N * 32); + T out = Zeros::get(); + if (this->warp_n_ == 0 && this->lane_ == 0) { +#pragma unroll + for (int it = 0; it < WARPS_N; it++) { + out = op(out, smem[it]); + } + } + return out; + } + + private: + T *smem0_; + T *smem1_; + bool use0_; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// +template +inline __device__ void +update_norm_stats(T &m_a, T &m2_a, T &n_a, T m_b, T m2_b, T n_b) { + const T n_ab = n_a + n_b; + + if constexpr (!isRMSNorm) { + const T rn_ab = (handleNan && (n_ab == Zeros::get())) ? Zeros::get() : (1.f / n_ab); + const T delta = m_a - m_b; + const float m2_ab = m2_a + m2_b + delta * delta * n_a * n_b * rn_ab; + const float m_ab = (n_a * m_a + n_b * m_b) * rn_ab; + n_a = n_ab; + m_a = m_ab; + m2_a = m2_ab; + } else { + const float m2_ab = m2_a + m2_b; + n_a = n_ab; + m_a = Zeros::get(); + m2_a = m2_ab; + } +} + +template +inline __device__ void +warp_chan_upd_dynamic(T &m_a, T &m2_a, T &n_a, int num_active) { + // Assume at least leftmost is valid and init: step = next_pow2(num_active) / 2 (might get NaN otherwise) + int highest_bit_set = (8 * sizeof(num_active)) - __clz(num_active - 1); + +#pragma unroll + for (int step = (1 << (highest_bit_set - 1)); step > 0; step /= 2) { + // Exchange + T n_b = utils::warp_shuffle_down(n_a, step); + T m_b = (!isRMSNorm ? utils::warp_shuffle_down(m_a, step) : Zeros::get()); + T m2_b = utils::warp_shuffle_down(m2_a, step); + update_norm_stats(m_a, m2_a, n_a, m_b, m2_b, n_b); + } + // Intra-warp broadcast (only lane 0 has valid stats). + m_a = __shfl_sync(uint32_t(-1), m_a, 0); + m2_a = __shfl_sync(uint32_t(-1), m2_a, 0); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// +// NOTE: WHOLE_CTA means if we'll use one empty mbarrier for inter-CTA sync using cluster. +// In the TMA LN engines it should be "true" since we're syncing the whole CTA with one empty mbarrier, while in the +// APEX LN engines it should be "false" as there's one empty mbarrier per warp m (and therefore WARPS_M empty mbarriers +// in total) +template +struct Stats { + // This could be done generically with the Reducer. But then we would have to exchange 3 instead of 2 fields. + + using InterCTASync = InterCTASync; + using BlockStats = Stats; + using stats_t = typename BlockStats::stats_t; + + enum { SMEM_BYTES = BlockStats::SMEM_BYTES }; + + template + inline __device__ + Stats(Params ¶ms, + uint32_t bidm, + uint32_t bidn, + uint32_t warp_m, + uint32_t warp_n, + uint32_t tidx, + uint32_t lane, + void *smem, + uint64_t *smem_bar) + : inter_cta_(params, bidm, bidn, tidx), + block_stats_(params, bidm, bidn, warp_m, warp_n, tidx, lane, smem, smem_bar), + bidn_(bidn), // CTA id within the group. + w0_(static_cast(params.workspace) + (bidm * WARPS_M + warp_m) * CTAS_PER_ROW), + w1_(w0_ + params.ctas_per_col * WARPS_M * CTAS_PER_ROW), + warp_n_(warp_n), + warp_m_(warp_m), + lane_(lane), + smem_bar_(smem_bar), + parity_(0) { + // the first several elements are the barriers + smem_cga_ = reinterpret_cast(&smem_bar[WHOLE_CTA? 2: (WARPS_M + 1)]); + } + + template + inline __device__ stats_t + compute(const T (&elts)[LDGS * NUM_ELTS], const T rn, int32_t warps_m) { + static constexpr uint32_t N = LDGS * NUM_ELTS; + constexpr T ELTS_PER_ROW_PER_CTA = N * WARPS_N * THREADS_PER_WARP; + // TODO rn is not really needed here.. + constexpr T block_rn = 1.f / T(ELTS_PER_ROW_PER_CTA); + stats_t block_stats = block_stats_.template compute(elts, block_rn, warps_m); + + if constexpr (USE_CLUSTER) { +#if (__CUDA_ARCH__ >= 900) && (CUDART_VERSION >= 12080) + auto cluster = cooperative_groups::this_cluster(); + + // Broadcast local results to other CTAs inside the CGA + // Size of smem_cga_: [WARPS_M][CTAS_PER_ROW] + // Thread 0 sends to block 0, Thread 1 sends to block 1, etc... + if ((warp_n_ == 0) && (lane_ < CTAS_PER_ROW)) { + st_async_remote(&smem_cga_[warp_m_ * CTAS_PER_ROW + bidn_], block_stats, lane_, &smem_bar_[WHOLE_CTA? 0: WARPS_M]); + } + + // Leader thread arrives on local barrier to indicate expected tx count + // We can't use (threadIdx.x == 0) directly here because it won't work for TMA + if ((warp_n_ == 0) && (warp_m_ == 0) && (lane_ == 0)) { + // It's possible that not all warp_m are active here, therefore, we can't use WARPS_M directly + uint32_t expected_tx_count = sizeof(block_stats) * CTAS_PER_ROW * warps_m; + utils::mbarrier_arrive_expect_tx_relaxed_cluster(&smem_bar_[WHOLE_CTA? 0: WARPS_M], expected_tx_count); + } + + // Wait on local barrier + while (!utils::mbarrier_try_wait_parity_relaxed_cluster(&smem_bar_[WHOLE_CTA? 0: WARPS_M], parity_)) {} + utils::fence_acquire_smem(); // Ensure we can read local smem values that are released by barrier flip + + T n = Zeros::get(); + T m = Zeros::get(); + T m2 = Zeros::get(); + if (lane_ < CTAS_PER_ROW) { + stats_t result = smem_cga_[warp_m_ * CTAS_PER_ROW + lane_]; + n = ELTS_PER_ROW_PER_CTA; + m = Get<0>::of(result); // should be 0 for RMSNorm + m2 = Get<1>::of(result); + } + utils::fence_release_smem(); // Ensure read of buffer remains ordered before smem_bar.arrive.relaxed +)KERNEL" + R"KERNEL( // Signal barrier is empty. + // Each thread must arrive on all the barriers (like an arrive broadcast). + // NOTE: here we only let each warp m arrive on the remote barriers of the same warp m (i.e. the same row), + // because if we have one empty barrier for the entire CTA, there's a difficulty how to init the count of + // the barrier (since some warps could be disabled here). The downside is we'll need a __syncthreads() here, + // which doesn't have obvious perf impact though. + // TODO: we could probably have one full bar per row (same as the empty bar) to get rid of the CTA sync, + // while the downside is more smem (mbarriers) will be needed. + if (!WHOLE_CTA) { + utils::namedBarrierSync(0, warps_m * WARPS_N * 32); + } + for (int other_block_rank = 0; other_block_rank < CTAS_PER_ROW; ++other_block_rank) { + uint64_t *remote_bar_empty = cluster.map_shared_rank(&smem_bar_[WHOLE_CTA? 1: warp_m_], other_block_rank); + utils::mbarrier_arrive_relaxed_cluster(remote_bar_empty); + } + + warp_chan_upd_dynamic(m, m2, n, CTAS_PER_ROW); + + // Wait for remote buffer to be writable again by waiting on local barrier: + while (!utils::mbarrier_try_wait_parity_relaxed_cluster(&smem_bar_[WHOLE_CTA? 1: warp_m_], parity_)) {} + utils::fence_acquire_smem(); // Order wrt st_async_remote + + // Flip parity bit for next iteration + parity_ ^= 1; + + return {m, m2}; +#else + static_assert(true, "Cluster enabled on host side but not available on device"); +#endif + } else { + // We switch workspace every iteration. + stats_t *workspace = inter_cta_.phase_counter_ & 0x1 ? w1_ : w0_; + + // Warp leaders 0 hold the CTA-local results. + if (warp_n_ == 0 && lane_ == 0) { + workspace[bidn_] = block_stats; + } + + // We need to check for the case where we are using CTA > 1 and WARP_M > 1 and N == 1 + // Because in this case we cannot rely on the __syncThreads in template::compute (because we will use the specialized template ::compute which does not have a sync) + // Bug 5221388: For WARPS_M > 1 and WARPS_N > 1, we also need the sync here, otherwise the stats received in + // `workspace[lane_]` may be NaN (as if some stats from the other CTAs are not initialized). + // Reason unknown yet and tracked in CL-16775. + if (CTAS_PER_ROW > 1 && WARPS_M > 1) { + utils::namedBarrierSync(0, warps_m * WARPS_N * 32); + } + + // Wait for all CTAS_PER_ROW CTAS in the group to have written their result. + inter_cta_.sync(warps_m * WARPS_N * 32); + + T n = Zeros::get(); + T m = Zeros::get(); + T m2 = Zeros::get(); + + // Every warp does the final reduction locally. + if constexpr (CTAS_PER_ROW <= THREADS_PER_WARP) { + if (lane_ < CTAS_PER_ROW) { + stats_t result = workspace[lane_]; + n = ELTS_PER_ROW_PER_CTA; + m = Get<0>::of(result); // should be 0 for RMSNorm + m2 = Get<1>::of(result); + } + warp_chan_upd_dynamic(m, m2, n, CTAS_PER_ROW); + } else { + static constexpr int32_t LOOP_PER_THREAD = (CTAS_PER_ROW + THREADS_PER_WARP - 1) / THREADS_PER_WARP; + + stats_t curr = workspace[lane_]; + n = ELTS_PER_ROW_PER_CTA; + m = Get<0>::of(curr); // should be 0 for RMSNorm + m2 = Get<1>::of(curr); + + // Collect stats for the current lane from ctas (#bidn % THREADS_PER_WARP == #lane) and then do the + // intra-warp reduction +#pragma unroll + for (int32_t loop_in_thread = 1; loop_in_thread < LOOP_PER_THREAD; ++loop_in_thread) { + const int32_t lane_in_loop = lane_ + loop_in_thread * THREADS_PER_WARP; + if (lane_in_loop < CTAS_PER_ROW) { + curr = workspace[lane_in_loop]; + T m_b = Get<0>::of(curr); + T m2_b = Get<1>::of(curr); + update_norm_stats(m, m2, n, m_b, m2_b, ELTS_PER_ROW_PER_CTA); + } + } + warp_chan_upd_dynamic(m, m2, n, THREADS_PER_WARP); + } + + return {m, m2}; + } + } + + InterCTASync inter_cta_; + BlockStats block_stats_; + + stats_t *w0_; + stats_t *w1_; + int bidn_; + int warp_n_; + int warp_m_; + int lane_; + + private: + stats_t *smem_cga_; + uint64_t *smem_bar_; + int parity_; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct Stats { + using WarpStats = Stats; + using stats_t = typename WarpStats::stats_t; + + enum { SMEM_BYTES = WARPS_M * WARPS_N * sizeof(stats_t) * 2 }; + + template + inline __device__ + Stats(Params ¶ms, + uint32_t bidm, + uint32_t bidn, + uint32_t warp_m, + uint32_t warp_n, + uint32_t tidx, + uint32_t lane, + void *smem, + uint64_t *smem_bar) + : warp_stats_(params, bidm, bidn, warp_m, warp_n, tidx, lane, smem, smem_bar), use0_(true) { + smem0_ = static_cast(smem) + warp_m * WARPS_N; + smem1_ = smem0_ + WARPS_M * WARPS_N; + } + + template + inline __device__ stats_t + compute(const T (&elts)[LDGS * NUM_ELTS], const T rn, int32_t warps_m) { + stats_t *smem = use0_ ? smem0_ : smem1_; + use0_ = !use0_; + // Compute warp local for all WARPS_N + static constexpr uint32_t N = LDGS * NUM_ELTS; + constexpr T warp_rn = 1.f / T(N * THREADS_PER_WARP); + stats_t warp_stats = warp_stats_.template compute(elts, warp_rn, warps_m); + + // Each warp warp leader stores its stats + const auto warp_n = warp_stats_.reducer_.warp_n_; + const auto lane = warp_stats_.reducer_.lane_; + if (lane == 0) { + smem[warp_n] = warp_stats; + } + utils::namedBarrierSync(0, warps_m * WARPS_N * 32); + + T n = Zeros::get(); + T m = Zeros::get(); + T m2 = Zeros::get(); + + if (lane < WARPS_N) { + stats_t result = smem[lane]; + n = N * THREADS_PER_WARP; + m = Get<0>::of(result); // should be 0 for RMSNorm + m2 = Get<1>::of(result); + } + + warp_chan_upd_dynamic(m, m2, n, WARPS_N); + + return {m, m2}; + } + + WarpStats warp_stats_; + stats_t *smem0_; + stats_t *smem1_; + bool use0_; +}; + +template +struct Stats { + using stats_t = typename TypeToVec2::Type; + // The simple Warp reducer. + using Reducer = Reducer; + + enum { SMEM_BYTES = 0 }; + + template + inline __device__ + Stats(Params ¶ms, + uint32_t bidm, + uint32_t bidn, + uint32_t warp_m, + uint32_t warp_n, + uint32_t tidx, + uint32_t lane, + void *smem, + uint64_t *smem_bar) + : reducer_(params, bidm, bidn, warp_m, warp_n, tidx, lane, smem, smem_bar) {} + + template + inline __device__ stats_t + compute(const T (&elts)[LDGS * NUM_ELTS], const T rn, int32_t warps_m) { + static constexpr uint32_t N = LDGS * NUM_ELTS; + auto sum = Sum(); + + T m = Zeros::get(); + if (!isRMSNorm) { +// CL-14115: The unroll factor 128 for LDGS was chosen based on the compilation/perf results for APEX LN_fwd engines +#pragma unroll(128 * NUM_ELTS) + for (int it = 0; it < N; it++) { + m += elts[it]; + } + m = reducer_.allreduce(m, sum, warps_m) * rn; + } + + T m2 = Zeros::get(); +#pragma unroll(128 * NUM_ELTS) + for (int it = 0; it < N; it++) { + T diff = (elts[it] - m); + m2 = __fmaf_ieee_rn(diff, diff, m2); + } + m2 = reducer_.allreduce(m2, sum, warps_m); + + return {m, m2}; + } + + Reducer reducer_; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// +//// Fast division helpers + +// Count leading zeros - start from most significant bit. +static int +clz(int32_t x) { + for (int32_t i = 31; i >= 0; --i) + if (((1 << i) & x) != 0) { + return 31 - i; + } + return 32; +} + +static int32_t +find_log_2(int32_t x, bool round_up = false) { + int32_t a = 31 - clz(x); + if (round_up) { + a += !(0 == (x & (x - 1))); + } + return a; +} + +static void +find_divisor(int32_t denom, uint32_t &mul_coeff, uint32_t &shift_coeff) { + if (denom == 0) { + return; + } + if (denom == 1) { + // if dividing by 1, reduced math doesn't work because mul_coeff would + // need to be 2^32, which doesn't fit into unsigned int. the div() + // routine handles this special case separately. + mul_coeff = 0; + shift_coeff = 0; + return; + } + // To express the division N/D in terms of a multiplication, what we first + // imagine is simply N*(1/D). However, 1/D will always evaluate to 0 (for D>1), + // so we need another way. There's nothing that says we have to use exactly + // the fraction 1/D; instead it could be any X/Y that reduces to 1/D (i.e., + // Y=X*D), or at least to "close enough" to it. If we pick Y that is a power + // of two, then the N*(X/Y) can be N*X followed by a right-shift by some amount. + // The power of two we should pick should be at least 2^32, because in the + // div() routine we'll use umulhi(), which returns only the upper 32 bits -- + // this being equivalent to a right-shift by 32. But we might want a higher + // power of two for better accuracy depending on the magnitude of the denominator. + // Once we've picked Y, then X [our mul_coeff value] is simply Y/D, rounding up, + // and we save shift_coeff as whatever further shift we have to do beyond + // what the umulhi() implies. + uint32_t p = 31 + find_log_2(denom, true); + uint32_t m = ((1ull << p) + static_cast(denom - 1)) / static_cast(denom); + mul_coeff = m; + shift_coeff = p - 32; +} + +__device__ __forceinline__ uint32_t +umulhi(uint32_t x, uint32_t y) { +#if defined(__CUDA_ARCH__) + return __umulhi(x, y); +#else + uint64_t z = static_cast(x) * y; + return static_cast(z >> 32); +#endif +} + +class reduced_divisor { + public: + reduced_divisor() {} + __forceinline__ + reduced_divisor(int32_t _y) + : y(_y) { + mul_coeff = 0U; + shift_coeff = 0U; + find_divisor(y, mul_coeff, shift_coeff); + } + __device__ __forceinline__ + reduced_divisor(uint32_t _mul_coeff, uint32_t _shift_coeff, int32_t _y) + : mul_coeff(_mul_coeff), shift_coeff(_shift_coeff), y(_y) {} + __device__ __forceinline__ int32_t + div(int32_t x) const { + // if dividing by 1, then find_divisor wouldn't have worked because + // mul_coeff would have had to be 2^32, which can't be represented, + // so we have to special case that one. + return (y != 1) ? umulhi(static_cast(x), mul_coeff) >> shift_coeff : x; + } + __device__ __forceinline__ int32_t + mod(int32_t x) const { + return x - (div(x) * y); + } + __device__ __forceinline__ void + divmod(int32_t x, int32_t &q, int32_t &mod) const { + q = div(x); + mod = x - (q * y); + } + __device__ __forceinline__ int32_t + get() const { + return y; + } + + protected: + uint32_t mul_coeff; + uint32_t shift_coeff; + int32_t y; +}; + + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct Kernel_traits_base { + using weight_t = weight_t_; + using input_t = input_t_; + using output_t = output_t_; + using compute_t = compute_t_; + using index_t = index_t_; + + enum { HIDDEN_SIZE = HIDDEN_SIZE_ }; + enum { THREADS_PER_CTA = THREADS_PER_CTA_ }; + enum { THREADS_PER_WARP = 32 }; +};)KERNEL" + R"KERNEL( + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template > +struct Kernel_traits : public Base { + using input_t = typename Base::input_t; + using weight_t = typename Base::weight_t; + using compute_t = typename Base::compute_t; + using output_t = typename Base::output_t; + using index_t = typename Base::index_t; + using norm_output_t = norm_output_t_; + enum { isRMSNorm = isRMSNorm_ }; + enum { isAdaLN = isAdaLN_ }; + enum { isBatchFirst = isBatchFirst_ }; + enum { hasGamma = hasGamma_ }; + enum { hasBeta = hasBeta_ }; + enum { CTAS_PER_ROW = CTAS_PER_ROW_ }; + enum { WARPS_M = WARPS_M_ }; + enum { WARPS_N = WARPS_N_ }; + enum { COLS = HIDDEN_SIZE_ }; + enum { HIDDEN_SIZE = HIDDEN_SIZE_ }; + enum { BATCH_SIZE = BATCH_SIZE_ }; + enum { BYTES_PER_LDG = BYTES_PER_LDG_ }; + enum { NUM_ELTS = BYTES_PER_LDG / sizeof(input_t) }; + enum { USE_GAMMA_SMEM = useGammaSmem_ }; + enum { USE_CLUSTER = useCluster_ }; + enum { WHOLE_CTA = wholeCTA_ }; + enum { THREADS_PER_ROW = WARPS_N * THREADS_PER_WARP }; + enum { THREADS_PER_CTA = WARPS_M * THREADS_PER_ROW }; + enum { ROWS_PER_WARP = ROWS_PER_WARP_ }; + enum { ROWS_PER_CTA = WARPS_M * ROWS_PER_WARP }; + + enum { BYTES_PER_ROW = COLS * sizeof(input_t) }; + enum { BYTES_PER_ROW_PER_CTA = THREADS_PER_ROW * BYTES_PER_LDG }; + // Multi-row per CTA not supported for multi-CTA => no smem for WGRAD needed + enum { SMEM_BYTES_WGRAD = ROWS_PER_CTA == 1 ? 0 : BATCH_SIZE * ROWS_PER_CTA * COLS * sizeof(compute_t) }; + + using reduce_t = typename TypeToVec2::Type; + using RMSReducer = Reducer; + using Reducer = Reducer; + + enum { SMEM_BYTES_DGRAD = Reducer::SMEM_BYTES }; + enum { SMEM_BYTES = SMEM_BYTES_DGRAD + SMEM_BYTES_WGRAD }; + + using Ivec = Vec; + using Ovec = Vec; + using Wvec = Vec; + using Cvec = Vec; + using NormOvec = Vec; + enum { ELTS_PER_LDG = BYTES_PER_LDG / sizeof(input_t) }; + + // Assume that each thread can handle the same number of elements in the output and weights as in the input. + // The number of columns fetched per load from input: one per thread. + enum { VEC_COLS_PER_LDG = CTAS_PER_ROW * THREADS_PER_ROW }; + // The total number of vectorized loads/stores per hidden vector. + enum { VEC_COLS = COLS / ELTS_PER_LDG }; + // The number of loads per thread for the input. + enum { LDGS = VEC_COLS / VEC_COLS_PER_LDG }; + + using Stats = Stats; + enum { SMEM_BYTES_FWD = Stats::SMEM_BYTES }; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + + +constexpr float CUDNN_FLT_MIN = 1.17549435082228750796873653722224568e-38F; // Minimum positive normalized float value +/** + * Base class to provide common block scaling utilities, private to `BlockScaleHelper` + */ +template +class BlockScaleHelperBase; + +/** + * Base class to provide common block scaling utilities, for nvfp4 + */ +template +class BlockScaleHelperBase { + public: + using scale_t = nv_fp8_e4m3; + + protected: + using compute_t = typename Cvec::Elt_type; + using output_t = typename Ovec::Elt_type; + + using index_t = uint32_t; + + static constexpr int32_t THREADS_PER_WARP = 32; + static constexpr int32_t BLOCK_SIZE = 16; + static constexpr int32_t NUM_ELTS = sizeof(Cvec) / sizeof(compute_t); + static constexpr int32_t GROUP_SIZE = BLOCK_SIZE / NUM_ELTS; + static constexpr int32_t SF_COUNT = THREADS_PER_WARP / GROUP_SIZE; + + static __device__ compute_t + scale(const compute_t value) { + constexpr compute_t nvfp4_max = 6.0f; + compute_t result = value / nvfp4_max; + // Clamp to CUDNN_FLT_MIN to match reference implementation and avoid 0 scaling factors + return fmaxf(result, CUDNN_FLT_MIN); + } + + static __device__ scale_t + fp32ToE4M3(float value) { + return scale_t(value); + } + + static __device__ Vec + vecDivideAndCast(const Cvec &input, compute_t scale) { + // Construct the nv_fp4x2_e2m1 outputs. + // If NUM_ELTS == 1, nv_fp4(ele1,0) + // If NUM_ELTS % 2 == 0; f2_i=nv_fp4(ele_i, ele_i+1) --> out[i/2] + // Per cuda_fp4.h and cuda_fp8.h + // The data type of __nv_fp4x2_storage_t is __nv_fp8_storage_t + // The data type of __nv_fp8_storage_t is unsigned char: + // typedef unsigned char __nv_fp8_storage_t; + // The reason why we use unsigned char is that the __shfl_xor_sync don't support nv_fp4 data type + Vec output; +#pragma unroll + for (int i = 0; i < NUM_ELTS; i += 2) { + float2 f2 = + make_float2((input.data.elt[i] / scale), (i + 1 < NUM_ELTS ? (input.data.elt[i + 1] / scale) : 0)); + output.data.elt[i / 2] = static_cast(nv_fp4x2_e2m1(f2).__x); + } + return output; + } +}; + +/** + * Base class to provide common block scaling utilities, for mxfp8 + */ +template +class BlockScaleHelperBase { + public: + using lne8m0_t = uint8_t; + using scale_t = lne8m0_t; + + protected: + using compute_t = typename Cvec::Elt_type; + using output_t = typename Ovec::Elt_type; + using index_t = uint32_t; + + static constexpr int32_t THREADS_PER_WARP = 32; + static constexpr int32_t BLOCK_SIZE = 32; + static constexpr int32_t NUM_ELTS = sizeof(Cvec) / sizeof(compute_t); + static constexpr int32_t GROUP_SIZE = BLOCK_SIZE / NUM_ELTS; + + static __device__ compute_t + scale(const compute_t value) { + static constexpr float FP8_MAX = std::is_same::value ? 57344.f : 448.f; + compute_t result = value / FP8_MAX; + // Clamp to CUDNN_FLT_MIN to match reference implementation and avoid 0 scaling factors + return fmaxf(result, CUDNN_FLT_MIN); + } + + template + static __device__ Ovec + vecDivideAndCast(const IOvec &input, compute_t scale) { + Ovec output; +#pragma unroll + for (int32_t i = 0; i < NUM_ELTS; ++i) { + output.data.elt[i] = output_t((compute_t(input.data.elt[i]) / scale)); + } + return output; + }; +}; + +/** + * Helper class to define the block scale operations + */ +template +class BlockScaleRowHelper; + +template +class BlockScaleColHelper; + +/** + * Specialization for MXFP8 rowwise scaling + */ +template +class BlockScaleRowHelper : private BlockScaleHelperBase { + private: + using Base = BlockScaleHelperBase; + using Base::BLOCK_SIZE; + using Base::GROUP_SIZE; + using Base::NUM_ELTS; + using Base::THREADS_PER_WARP; + using typename Base::compute_t; + using typename Base::index_t; + using typename Base::scale_t; + + public: + __device__ + BlockScaleRowHelper() = default; + + template + __device__ void + blockQuantizeStore(const IOvec &z_math, void *sf_ptr, index_t sf_idx, void *z_ptr, index_t z_idx) { + const index_t group_id = lane / GROUP_SIZE; + + // 1. compute scaling factor + compute_t sf = 0; +#pragma unroll + // compute the amax across NUM_ELTS elements + for (int jt = 0; jt < NUM_ELTS; jt++) { + sf = fmaxf(sf, fabsf(compute_t(z_math.data.elt[jt]))); + } + +// Currently disabled for rowwise block scaling as perf regression observed +#if 0 && (defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 1000) && defined(__CUDA_ARCH_FEAT_SM100_ALL) +#pragma unroll + for (int jt = 0; jt < NUM_ELTS; jt++) { + if (group_id == jt) { + asm volatile("redux.sync.max.f32 %0, %1, %2;" : "=f"(sf) : "f"(sf), "r"(__activemask())); + } + } +#else + // warp reduce (binary reduction) starting from GROUP_SIZE / 2 to aggregate + // a total of NUM_ELTS * GROUP_SIZE = BLOCK_SIZE number of z values in sf +#pragma unroll + for (int offset = GROUP_SIZE / 2; offset > 0; offset /= 2) { + sf = fmaxf(sf, __shfl_down_sync(0xffffffff, sf, offset)); + } + + // broadcast from the lane with the valid values to the other threads in the group + sf = __shfl_sync(0xffffffff, sf, group_id * GROUP_SIZE); +#endif + // scale the row amax to get the scaling factor + sf = Base::scale(sf); + + // 2. store scaling factor + // Warp orchestration to compute scale_row: + // e.g. for NUM_ELTS == 2: + // | W0SF0 W0SF1 | W0SF0 W0SF1 | ... | W0SF0 W0SF1 | W0SF0 W0SF1 |... + // | W1SF0 W1SF1 | W1SF0 W1SF1 | ... | W1SF0 W1SF1 | W1SF0 W1SF1 |... + // | ... | ... | ... | ... | ... |... + // | W31SF0 W31SF1 | W31SF0 W31SF1 | ... | W31SF0 W31SF1 | W31SF0 W31SF1 |... + // |---------------|---------------|-----|--------------------------|---------------|... + // |----- CTA0 ----|----- CTA1 ----| ... |--- CTA --|----- CTA0 ----|... + // | | + // |-------------------------------LDG0-----------------------------|-------LDG1-----... + Vec sf_out; + // only correct for group_id == 0, but it's ok because only lane == 0 is storing sf_out + sf_out.data.elt[0] = float_to_e8m0(sf); +#pragma unroll + for (int jt = 1; jt < NUM_ELTS; jt++) { + sf_out.data.elt[jt] = float_to_e8m0(__shfl_sync(0xffffffff, sf, jt * GROUP_SIZE)); + } + if (lane == 0) { + sf_out.store_to(sf_ptr, sf_idx); + } + // No need to check for 0x00 - scale() now clamps to FLT_MIN which converts to 0x01 + float scale = e8m0_to_float(sf_out.data.elt[0]); + Ovec z_row_scaled = Base::vecDivideAndCast(z_math, scale); + z_row_scaled.store_to(z_ptr, z_idx); + } + + private: + const index_t lane = threadIdx.x % THREADS_PER_WARP; +}; + +/** + * Specialization for MXFP8 colwise scaling in 1D2X2X + */ +template +class BlockScaleColHelper : private BlockScaleHelperBase { + private: + using Base = BlockScaleHelperBase; + using Base::BLOCK_SIZE; + using Base::GROUP_SIZE; + using Base::NUM_ELTS; + using Base::THREADS_PER_WARP; + using typename Base::compute_t; + using typename Base::index_t; + using typename Base::output_t; + using typename Base::scale_t; + + public: + __device__ + BlockScaleColHelper(float *mxfp8_tile) + : mxfp8_tile_(mxfp8_tile) {} + + template + __device__ void + initTile(const IOvec &z_math, int num_threads) {)KERNEL" + R"KERNEL( return initTile(z_math, warp, num_threads); + } + + template + __device__ void + initTile(const IOvec &z_math, index_t row_in_tile, int num_threads) { + constexpr int entry_barrier_arbitrary = 5; + utils::namedBarrierSync(entry_barrier_arbitrary, num_threads); +#pragma unroll + for (index_t jt = 0; jt < NUM_ELTS; jt++) { + index_t idx_in_mxfp8_smem = + (jt * (BLOCK_SIZE + /*padding row*/ 1) + row_in_tile) * MXFP8_1D2X2X_SMEM_COLS + lane; + mxfp8_tile_[idx_in_mxfp8_smem] = float(z_math.data.elt[jt]); + } + } + + template + __device__ void + blockQuantizeStore(void *sf_ptr, + index_t sf_row_idx, + index_t sf_col_idx, + index_t sf_row_width, + void *z_ptr, + index_t z_idx, + int num_threads) { + return blockQuantizeStore( + sf_ptr, sf_row_idx, sf_col_idx, sf_row_width, z_ptr, z_idx, num_threads, warp); + } + + template + __device__ void + blockQuantizeStore(void *sf_ptr, + index_t sf_row_idx, + index_t sf_col_idx, + index_t sf_row_width, + void *z_ptr, + index_t z_idx, + int num_threads, + index_t row_in_tile) { + // Ensure all temp outputs have been written to mxfp8 smem for the current ldg + constexpr int entry_barrier_arbitrary = 4; + utils::namedBarrierSync(entry_barrier_arbitrary, num_threads); + const index_t group_id = lane / GROUP_SIZE; + +#if (defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 1000) && defined(__CUDA_ARCH_FEAT_SM100_ALL) + // For sm_100a arch, we can use redux.sync.op{.abs.}{.NaN}.f32 instruction to reduce the min/max value + // https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-redux-sync + compute_t amax_vals[NUM_ELTS]; +#pragma unroll + for (int jt = 0; jt < NUM_ELTS; jt++) { + const index_t idx_in_mxfp8_smem = jt * NEXT_COL_SMEM_OFFSET + lane * MXFP8_1D2X2X_SMEM_COLS + row_in_tile; + asm volatile("redux.sync.max.abs.f32 %0, %1, 0xffffffff;" + : "=f"(amax_vals[jt]) + : "f"(mxfp8_tile_[idx_in_mxfp8_smem])); + } + compute_t sf = amax_vals[group_id]; +#else + // 1. compute scaling factor + compute_t sf = 0; +#pragma unroll + // compute the amax across NUM_ELTS elements in the same col (in shared memory, adjacent rows) + for (int jt = 0; jt < NUM_ELTS * MXFP8_1D2X2X_SMEM_COLS; jt += MXFP8_1D2X2X_SMEM_COLS) { + const index_t idx_in_mxfp8_smem = ROW_SMEM_OFFSET + jt + row_in_tile; + sf = fmaxf(sf, fabsf(mxfp8_tile_[idx_in_mxfp8_smem])); + } + + // warp reduce (binary reduction) starting from GROUP_SIZE / 2 to aggregate + // a total of NUM_ELTS * GROUP_SIZE = BLOCK_SIZE number of z values in sf +#pragma unroll + for (int offset = GROUP_SIZE / 2; offset > 0; offset /= 2) { + sf = fmaxf(sf, __shfl_down_sync(0xffffffff, sf, offset)); + } + + // broadcast from the lane with the valid values to the other threads in the group + sf = __shfl_sync(0xffffffff, sf, group_id * GROUP_SIZE); +#endif + + // scale the col amax to get the scaling factor + sf = Base::scale(sf); + + // Warp orchestration to compute scale_col: + // e.g. for NUM_ELTS == 2: + // | W0SF0 W0SF1 W1SF0 W1SF1 ... W31SF0 W31SF1 | W0SF0 W0SF1 ... W31SF0 W31SF1 |... + // |----warp0---|---warp1--| ... |----warp31---|----warp0---|... |----warp31---|... + // |--------------------CTA0-------------------|---------------CTA1------------|... + // |------------------------------------------LDG0------------------------------... + Ovec z_col_scaled; + if constexpr (!Transpose) { + // 2. store scaling factor + // different columns require different sf, so all threads must exchange sf + Vec sf_out; +#pragma unroll + for (int jt = 0; jt < NUM_ELTS; jt++) { + sf_out.data.elt[jt] = float_to_e8m0(__shfl_sync(0xffffffff, sf, jt * GROUP_SIZE)); + } + if (lane == 0) { + sf_out.store_to(sf_ptr, sf_row_idx * sf_row_width + sf_col_idx); + } + + // 3. scale and store output +#pragma unroll + for (int jt = 0; jt < NUM_ELTS; jt++) { + // iterate over NUM_ELTS elements in a row (in shared memory, every other NUM_ELTS in a column) + index_t z_idx_in_mxfp8_smem = jt * NEXT_COL_SMEM_OFFSET + lane * MXFP8_1D2X2X_SMEM_COLS + row_in_tile; + // No need to check for 0x00 - scale() now clamps to FLT_MIN which converts to 0x01 + float scale = e8m0_to_float(sf_out.data.elt[jt]); + z_col_scaled.data.elt[jt] = + output_t(mxfp8_tile_[z_idx_in_mxfp8_smem] / scale); + } + } else { + // 2. store scaling factor + uint8_t sf_e8m0 = float_to_e8m0(sf); + // No need to check for 0x00 - scale() now clamps to FLT_MIN which converts to 0x01 + if (lane % GROUP_SIZE == 0) { + static_cast(sf_ptr)[(sf_row_idx + group_id) * sf_row_width + sf_col_idx] = sf_e8m0; + } + + // 3. scale and store output + float scale = e8m0_to_float(sf_e8m0); + // No need to check for 0.0f - scale() now clamps to FLT_MIN +#pragma unroll + for (int jt = 0; jt < NUM_ELTS; jt++) { + // iterate over NUM_ELTS elements in a column (in shared memory, adjacent NUM_ELTS rows) + index_t z_idx_in_mxfp8_smem = ROW_SMEM_OFFSET + jt * MXFP8_1D2X2X_SMEM_COLS + row_in_tile; + z_col_scaled.data.elt[jt] = output_t(mxfp8_tile_[z_idx_in_mxfp8_smem] / scale); + } + } + z_col_scaled.store_to(z_ptr, z_idx); + + } + + private: + static constexpr index_t MXFP8_1D2X2X_SMEM_ROWS = BLOCK_SIZE * NUM_ELTS + /*padding rows between*/ (NUM_ELTS - 1); + static constexpr index_t MXFP8_1D2X2X_SMEM_COLS = BLOCK_SIZE + /*padding col*/ 1; + static constexpr index_t NEXT_COL_SMEM_OFFSET = (BLOCK_SIZE + /*padding row*/ 1) * MXFP8_1D2X2X_SMEM_COLS; + // For MXFP8 1d2x2x, we store the 32x32xNUM_ELTS tile in shared memory to compute the col amax for scaling within + // the current CTA + // e.g. for NUM_ELTS == 2, the shared memory layout for 1d2x2x is as follows: + // + // col0 col2 col4 col6 ... col62 + // ↓ ↓ ↓ ↓ ↓ + // |-------------------------------------------| + // | T0V0 | T1V0 | T2V0 | T3V0 |...| T31V0 | 0 | <-- warp0 to store at jt=0 + // | T0V0 | T1V0 | T2V0 | T3V0 |...| T31V0 | 0 | <-- warp1 to store at jt=0 + // | .. | .. | .. | .. |...| ... | 0 | <-- ... + // | T0V0 | T1V0 | T2V0 | T3V0 |...| T31V0 | 0 | <-- warp31 to store at jt=0 + // |--------------------|------|---|-------|---| + // | 0 | 0 | 0 | 0 |...| 0 | 0 | <-- padding row between to avoid bank conflicts when col-scaling + // |--------------------|------|---|-------|---| + // | T0V1 | T1V1 | T2V1 | T3V1 |...| T31V1 | 0 | <-- warp0 to store at jt=1 + // | T0V1 | T1V1 | T2V1 | T3V1 |...| T31V1 | 0 | <-- warp1 to store at jt=1 + // | .. | .. | .. | .. |...| ... | 0 | <-- ... + // | T0V1 | T1V1 | T2V1 | T3V1 |...| T31V1 | 0 | <-- warp31 to store at jt=1 + // |-------------------------------------------| + // ↑ ↑ ↑ ↑ ↑ ↑ + // col1 col3 col5 col7 ... col63 padding col to avoid bank conflicts when computing col-scale factors + // + // ↑ ↑ ↑ ↑ ↑ + // warp0 warp1 warp2 warp3 ... warp31 + // to compute and apply the col-scale factors + float *mxfp8_tile_ = nullptr; + + const index_t lane = threadIdx.x % THREADS_PER_WARP; + const index_t warp = threadIdx.x / THREADS_PER_WARP; + + const index_t ROW_SMEM_OFFSET = ((lane * NUM_ELTS) + /*padding rows*/ (lane / GROUP_SIZE)) * MXFP8_1D2X2X_SMEM_COLS; +}; + +/** + * Specialization for NVFP4 rowwise scaling + */ +template +class BlockScaleRowHelper : private BlockScaleHelperBase { + using Base = BlockScaleHelperBase; + using Base::BLOCK_SIZE; + using Base::GROUP_SIZE; + using Base::NUM_ELTS; + using Base::SF_COUNT; + using Base::THREADS_PER_WARP; + using typename Base::compute_t; + using typename Base::index_t; + using typename Base::output_t; + using typename Base::scale_t; + + public: + __device__ + BlockScaleRowHelper() = default; + + __device__ void + blockQuantizeStore(const Cvec &z_math, void *sf_ptr, index_t sf_idx, void *z_ptr, index_t z_idx) { + // 1. compute scaling factor + compute_t sf = 0; +#pragma unroll + // compute the amax across NUM_ELTS elements + for (int jt = 0; jt < NUM_ELTS; jt++) { + sf = fmaxf(sf, fabsf(z_math.data.elt[jt])); + } + + // warp reduce (binary reduction) starting from GROUP_SIZE / 2 to aggregate + // a total of NUM_ELTS * GROUP_SIZE = BLOCK_SIZE number of z values in sf +#pragma unroll + for (int offset = GROUP_SIZE / 2; offset > 0; offset /= 2) { + sf = fmaxf(sf, __shfl_down_sync(0xffffffff, sf, offset)); + } + + // broadcast from the lane with the valid values to the other threads in the group + const index_t group_id = lane / GROUP_SIZE; + sf = __shfl_sync(0xffffffff, sf, group_id * GROUP_SIZE); + + // scale the row amax to get the scaling factor + sf = Base::scale(sf); + + // 2. store scaling factor + Vec sf_out; + // only correct for group_id == 0, but it's ok because only lane == 0 is storing sf_out + sf_out.data.elt[0] = Base::fp32ToE4M3(sf); +#pragma unroll + for (int jt = 1; jt < SF_COUNT; jt++) { + sf_out.data.elt[jt] = Base::fp32ToE4M3(__shfl_sync(0xffffffff, sf, jt * GROUP_SIZE)); + } + if (lane == 0) { + sf_out.store_to(sf_ptr, sf_idx); + } + + // TODO: can consider doing something like the following to remove the need for a shuffle loop + // template + // struct UintContainer { + // using type = std::conditional_t>>>; // void for unsupported sizes + // }; + // Base::vecDivideAndCast returns Vec, 2> + // 3. scale and store output + auto z_row_scaled = Base::vecDivideAndCast(z_math, sf); +#pragma unroll + // Consturct z_row_scaled by combining the adjacent thread's z_row_scaled + // Vec z_row_scaled + for (int i = 0; i < NUM_ELTS / 2; i++) { + uint8_t adjacent_val = __shfl_xor_sync(0xffffffff, z_row_scaled.data.elt[i], 1); + z_row_scaled.data.elt[i + NUM_ELTS / 2] = adjacent_val; + } + if (NUM_ELTS == 1) { + uint8_t adjacent_val = __shfl_xor_sync(0xffffffff, z_row_scaled.data.elt[0], 1); + z_row_scaled.data.elt[0] |= adjacent_val << 4U; + } + if (lane % 2 == 0) { + z_row_scaled.store_to(z_ptr, z_idx / 2); + } + } + + private: + const index_t lane = threadIdx.x % THREADS_PER_WARP; +}; + +)KERNEL"; +inline constexpr size_t ln_headers_source_len = sizeof(ln_headers_source) - 1; + +} // namespace generated +} // namespace experimental +} // namespace cudnn_frontend + +#if defined(_MSC_VER) +#pragma warning(pop) +#endif + +#if defined(__clang__) +#pragma clang diagnostic pop +#endif diff --git a/deps/cudnn-frontend/include/cudnn_frontend/generated/sdpa/sm100/prefill/full_seqlens/d128_fprop_kernel.h b/deps/cudnn-frontend/include/cudnn_frontend/generated/sdpa/sm100/prefill/full_seqlens/d128_fprop_kernel.h new file mode 100644 index 0000000..ee3664a --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/generated/sdpa/sm100/prefill/full_seqlens/d128_fprop_kernel.h @@ -0,0 +1,3479 @@ +// Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: MIT +#pragma once +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Woverlength-strings" +#endif +#if defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable : 4068) +#endif +namespace cudnn_frontend { +namespace experimental { +namespace generated { +inline constexpr const char sm100_d128_fprop_source[] = + R"KERNEL( + +// receive_op 0 includes +typedef unsigned long long uint64_t; +typedef unsigned int uint32_t; +typedef unsigned short uint16_t; +typedef unsigned char uint8_t; +typedef long long int64_t; +typedef int int32_t; +typedef short int16_t; +typedef signed char int8_t; +typedef unsigned int r32; + +struct CUtensorMap { + alignas(64) uint64_t opaque[16]; +}; + +#define __FLT_MIN__ 1.17549435082228750796873653722224568e-38F +#define __FLT_MAX__ 3.40282346638528859811704183484516925e+38F + +#define CUDACC_VERSION __CUDACC_VER_MAJOR__ * 10 + __CUDACC_VER_MINOR__ + +namespace fort { +static const uint64_t AMPERE_MEM_DESC_DEFAULT = uint64_t(0x1000000000000000ul); +static const uint64_t MEM_DESC_DEFAULT = AMPERE_MEM_DESC_DEFAULT; +static const uint32_t MMA_PEER_BIT_MASK = 0xFEFFFFFF; + +#define NAN __int_as_float(0x7fffffff) +#define POS_INFINITY __int_as_float(0x7f800000) +#define NEG_INFINITY __int_as_float(0xff800000) + +typedef uint16_t bfloat16_t; + +#define FORT_MIN(a, b) ((a) < (b) ? (a) : (b)) +#define FORT_MAX(a, b) ((a) > (b) ? (a) : (b)) +#define FORT_DIV_UP(a, b) (((a) + (b) - 1) / (b)) +#define FORT_ROUND_UP(a, b) ((((a) + (b) - 1) / (b)) * (b)) + +typedef struct tensor_descriptor { + static const int MAX_DIMS = 12; + + int64_t num_dims; + int64_t dims[MAX_DIMS]; + int64_t strides[MAX_DIMS]; +} tensor_descriptor; + +typedef struct FastDivisor { + uint32_t val, shr, mul; +} FastDivisor_t; + +inline __device__ void fastDivMod(const FastDivisor_t &d, uint32_t val, + uint32_t &div, uint32_t &mod) { + div = __umulhi((uint32_t)2 * val, d.mul) >> d.shr; + mod = val - div * d.val; +} + +inline __device__ char *get_smem_loc_epilogue_swizzle_128b( + char *smem_addr, int local_block_id, int tid, int local_row, int column, + size_t element_size, int block_size, int row_per_tile) { + if (element_size * block_size == 256) { + // Case 1: 2 * swizzle tile size = block size -> half block per swizzle tile + int swizzle_col = local_row ^ (column % 8); + + // (local_block_id * 2 + (column / 8)) * row_per_tile * 128: gives offset of + // corresponding swizzle tile tid * 128: tid gives number of loads in block + // swizzle_col * 16B: swizzle_col gives transformed column + return smem_addr + + (local_block_id * 2 + (column / 8)) * row_per_tile * 128 + + tid * 128 + swizzle_col * 16; + } else if (element_size * block_size == 128) { + // Case 2: swizzle tile size = block size -> one block per swizzle tile + int swizzle_col = local_row ^ (column % 8); + + // local_block_id * row_per_tile * 128: gives offset of corresponding + // swizzle tile tid * 128: tid gives number of loads in block swizzle_col * + // 16B: swizzle_col gives transformed column + return smem_addr + local_block_id * row_per_tile * 128 + tid * 128 + + swizzle_col * 16; + } else { + // Case 3: swizzle tile size / 2 = block size -> two blocks per swizzle tile + int swizzle_col = local_row ^ ((column + local_block_id * 4) % 8); + + // (local_block_id / 2) * row_per_tile * 128: gives offset of corresponding + // swizzle tile tid * 128: tid gives number of loads in block swizzle_col * + // 16B: swizzle_col gives transformed column + return smem_addr + (local_block_id / 2) * row_per_tile * 128 + tid * 128 + + swizzle_col * 16; + } +} + +inline __device__ static void get_next_block_id(uint32_t result, + uint32_t mbar) { +#if __CUDA_ARCH__ >= 1200 && __CUDA_ARCH__ < 1300 + asm volatile("{clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::" + "complete_tx::bytes.b128 [%0], [%1];}\n\t" ::"r"(result), + "r"(mbar)); +#else + asm volatile( + "{clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_" + "tx::bytes.multicast::cluster::all.b128 [%0], [%1];}\n\t" ::"r"(result), + "r"(mbar)); +#endif +} + +inline __device__ float2 ffma2(const float2 &a, const float2 &b, + const float2 &c) { + uint64_t d; + asm volatile("fma.rn.f32x2 %0, %1, %2, %3;\n" + : "=l"(d) + : "l"(reinterpret_cast(a)), + "l"(reinterpret_cast(b)), + "l"(reinterpret_cast(c))); + return reinterpret_cast(d); +} + +inline __device__ float2 fadd2(const float2 &a, const float2 &b) { + uint64_t c; + asm volatile("add.f32x2 %0, %1, %2;\n" + : "=l"(c) + : "l"(reinterpret_cast(a)), + "l"(reinterpret_cast(b))); + return reinterpret_cast(c); +} + +inline __device__ float2 fmul2(const float2 &a, const float2 &b) { + uint64_t c; + asm volatile("mul.f32x2 %0, %1, %2;\n" + : "=l"(c) + : "l"(reinterpret_cast(a)), + "l"(reinterpret_cast(b))); + return reinterpret_cast(c); +} + +inline __device__ float fmax3(float a, float b, float c) { +#if (__CUDACC_VER_MAJOR__ >= 13) + float d; + asm volatile("max.ftz.f32 %0, %1, %2, %3;\n" + : "=f"(d) + : "f"((a)), "f"((b)), "f"((c))); + return d; +#else + #error "This kernel is not supported for CUDA toolkit version < 13.0" + return 0.0f; +#endif +} + +inline __device__ float row_max_reduction_128_elems(uint32_t reg[128]) { + float tmp_max_0[42]; + float tmp_max_1[14]; + float tmp_max_2[5]; + float tmp_max_3[2]; + +#pragma unroll + for (int i = 0; i < 126; i += 3) { + tmp_max_0[i / 3] = fmax3(reinterpret_cast(reg[i + 0]), + reinterpret_cast(reg[i + 1]), + reinterpret_cast(reg[i + 2])); + } +#pragma unroll + for (int i = 0; i < 42; i += 3) { + tmp_max_1[i / 3] = + fmax3(tmp_max_0[i + 0], tmp_max_0[i + 1], tmp_max_0[i + 2]); + } +#pragma unroll + for (int i = 0; i < 12; i += 3) { + tmp_max_2[i / 3] = + fmax3(tmp_max_1[i + 0], tmp_max_1[i + 1], tmp_max_1[i + 2]); + } + tmp_max_2[4] = + fmax3(tmp_max_1[12], tmp_max_1[13], reinterpret_cast(reg[126])); + tmp_max_3[0] = fmax3(tmp_max_2[0], tmp_max_2[1], tmp_max_2[2]); + tmp_max_3[1] = + fmax3(tmp_max_2[3], tmp_max_2[4], reinterpret_cast(reg[127])); + + return fmax3(tmp_max_3[0], tmp_max_3[1], tmp_max_3[1]); +} + +#if __CUDA_ARCH__ >= 900 + +inline __device__ uint32_t elect_one_sync() { + uint32_t pred = 0; + uint32_t laneid = 0; + asm volatile("{\n" + ".reg .b32 %%rx;\n" + ".reg .pred %%px;\n" + " elect.sync %%rx|%%px, %2;\n" + "@%%px mov.s32 %1, 1;\n" + " mov.s32 %0, %%rx;\n" + "}\n" + : "+r"(laneid), "+r"(pred) + : "r"(0xFFFFFFFF)); + return pred; +} + +template inline __device__ void reg_alloc() { + // const int TARGET_REG_COUNT = 232; // Example values (use with reg_delloc's + // VALUE together): 224, 232, 208, 216 + asm volatile("setmaxnreg.inc.sync.aligned.u32 %0;\n" + : + : "n"(TARGET_REG_COUNT)); +} + +template inline __device__ void reg_dealloc() { + // const int TARGET_REG_COUNT = 40; // Exmaple values: 56, 40, 88, 72 + asm volatile("setmaxnreg.dec.sync.aligned.u32 %0;\n" + : + : "n"(TARGET_REG_COUNT)); +} + +inline __device__ void named_barrier_arrive(uint32_t BARRIER_ID, + uint32_t NUM_THREADS) { + asm volatile("bar.arrive %0, %1;" : : "r"(BARRIER_ID), "r"(NUM_THREADS)); +} + +inline __device__ void named_barrier_wait(uint32_t BARRIER_ID, + uint32_t NUM_THREADS) { + asm volatile("bar.sync %0, %1;" ::"r"(BARRIER_ID), "r"(NUM_THREADS)); +} + +inline __device__ void tmastg_arrive() { + asm volatile("cp.async.bulk.commit_group;"); +} + +template inline __device__ void tmastg_wait_count() { + asm volatile("cp.async.bulk.wait_group.read %0;" : : "n"(Count) : "memory"); +} + +#endif + +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 +inline __device__ void smem_bar_init(uint32_t smem_ptr, uint32_t thread_count) { + asm volatile("mbarrier.init.shared::cta.b64 [%0], %1;\n" ::"r"(smem_ptr), + "r"(thread_count)); +} + +inline __device__ void +smem_bar_set_transaction_count(uint32_t smem_ptr, uint32_t expected_copy_bytes, + uint32_t pred = 0) { + asm volatile("{\n\t.reg .pred p;" + " \n\tsetp.eq.u32 p, %2, 1;" + " \n\t@p mbarrier.arrive.expect_tx.shared::cta.b64 _, [%0], %1;" + " \n\t}" ::"r"(smem_ptr), + "r"(expected_copy_bytes), "r"(pred)); +} + +inline __device__ uint32_t smem_bar_peek(uint32_t smem_ptr, + uint32_t bar_phase) { + uint32_t bar_phase_out; + asm volatile("{\n\t.reg .pred P1;" + " \n\tmbarrier.try_wait.parity.shared::cta.b64 P1, [%1], %2;" + " \n\tselp.b32 %0, 1, 0, P1; \n\t}" + : "=r"(bar_phase_out) + : "r"(smem_ptr), "r"(bar_phase)); + return bar_phase_out; +} + +inline __device__ void +wait_barrier(uint32_t smem_int_ptr, + int phase_bit) // Current phase bit the barrier waiting to flip +{ + asm volatile("{\n" + ".reg .pred P1;\n" + "LAB_WAIT:\n" + "mbarrier.try_wait.parity.shared::cta.b64 P1, [%0], %1;\n" + "@P1 bra.uni DONE;\n" + "bra.uni LAB_WAIT;\n" + "DONE:\n" + "}\n" ::"r"(smem_int_ptr), + "r"(phase_bit)); +} + +// Barrier arrive on local smem +inline __device__ void arrive_barrier(uint32_t smem_addr) { + asm volatile("{\n\t" + "mbarrier.arrive.shared::cta.b64 _, [%0];\n\t" + "}" + : + : "r"(smem_addr)); +} + +#endif + +// Circular Buffer Index + Associated Phase +// Assumes only one operation possible - i.e., ++ +template struct PipelineState { +public: + static constexpr int Stages = Stages_; + + inline __device__ PipelineState(int index, int state) + : state_(state), index_(index) {} + + inline __device__ int phase() const { return state_; } + + inline __device__ int index() const { return index_; } + + inline __device__ void operator++() { + if (Stages > 1) { + ++index_; + if (index_ == Stages) { + index_ = 0; + state_ ^= 1; + } + } else { + state_ ^= 1; + } + } + + int state_; + int index_; +}; + +template +inline __device__ constexpr uint64_t build_utcmma_instruction_desc() { + return (static_cast(0) | (MAX_SHIFT_FOR_WS << 30) | + ((UTCMMA_M >> 5) << 25) | ((UTCMMA_N >> 3) << 17) | + (B_TRANSPOSE << 16) | (A_TRANSPOSE << 15) | (B_NEGATE << 14) | + (A_NEGATE << 13) | (SRC_B_TYPE << 10) | (SRC_A_TYPE << 7) | + (SPARSE_METADATA_FORMAT << 6) | (ACC_TYPE << 4) | + (SATURATE_ENABLED << 3) | (SPARSE_ENABLED << 2) | + (SPARSE_METADATA_ID2 << 0)) + << 32; +} +)KERNEL" + R"KERNEL( + +class Smem_utcmma_descriptor { +public: + inline __device__ Smem_utcmma_descriptor(uint64_t SWIZZLE_MODE, + uint64_t BASE_OFFSET, + uint64_t DESC_VERSION, + uint64_t BYTES_PER_LEADING_DIM, + uint64_t BYTES_PER_STRIDE_DIM) { + // ------------------------------------ + // Setup smem descriptor for operand A: + // ------------------------------------ + // Note 1: SWIZZLE_NONE = 0, SWIZZLE_128B = 2, SWIZZLE_64B = 4, SWIZZLE_32B + // = 6, SWIZZLE_128B_ATOM32B = 1, N/A = 3, N/A = 5, N/A = 7 + const uint64_t SWIZZLE_MODE_IN_BIT_LOCATION = SWIZZLE_MODE + << 61; // bits: 63-61 + + // Note 2: Base offset. Valid only for matrix descriptor 1, 2 or 3, 4 + const uint64_t BASE_OFFSET_IN_BIT_LOCATION = BASE_OFFSET + << 49; // bits 51-49 + + // Note 3: Descriptor version, needs to be set to 1. ??? + const uint64_t DESC_VERSION_IN_BIT_LOCATION = DESC_VERSION + << 46; // bits 48-46 + + // Note 4: Stride dimension byte offset, 16 byte aligned, 4 LSBs not + // included + const uint64_t STRIDE_DIM_BYTE_OFFSET = BYTES_PER_STRIDE_DIM >> 4; + const uint64_t STRIDE_DIM_BYTE_OFFSET_IN_BIT_LOCATION = + STRIDE_DIM_BYTE_OFFSET << 32; // bits 45-32 + + // Note 5: Leading dimension byte offset, 16 byte aligned, 4 LSBs not + // included + const uint64_t LEADING_DIM_BYTE_OFFSET = BYTES_PER_LEADING_DIM >> 4; + const uint64_t LEADING_DIM_BYTE_OFFSET_IN_BIT_LOCATION = + LEADING_DIM_BYTE_OFFSET << 16; // bits 29-16 + + desc = + (SWIZZLE_MODE_IN_BIT_LOCATION | BASE_OFFSET_IN_BIT_LOCATION | + DESC_VERSION_IN_BIT_LOCATION | STRIDE_DIM_BYTE_OFFSET_IN_BIT_LOCATION | + LEADING_DIM_BYTE_OFFSET_IN_BIT_LOCATION); + } + template + inline __device__ void set_smem(uint32_t smem) { + int2 &tmp = reinterpret_cast(desc); + tmp.x |= (static_cast(smem & 0xFFFFFF) >> 4); + + int2 &tmp_initial = reinterpret_cast(initial_desc); + tmp_initial.x = tmp.x; + + max_desc = tmp.x + (BYTES_PER_BUFFER >> 4) * (BUFFER_COUNT - 1); + } + + template + inline __device__ void increment_smem_buffer() { + int2 &tmp = reinterpret_cast(desc); + tmp.x += (tmp.x >= max_desc) ? -(BYTES_PER_BUFFER >> 4) * (BUFFER_COUNT - 1) + : (BYTES_PER_BUFFER >> 4); + } + + template + inline __device__ void stage_increment_smem_buffer(int stage) { + int2 &tmp_initial = reinterpret_cast(initial_desc); + int2 &tmp = reinterpret_cast(desc); + + tmp.x = tmp_initial.x + (BYTES_PER_BUFFER >> 4) * stage; + } + + template inline __device__ void add_smem_offset() { + int2 &tmp = reinterpret_cast(desc); + tmp.x += (BYTES_OFFSET >> 4); + } + + uint64_t initial_desc; + uint64_t desc; + uint32_t max_desc; +}; + +// cta1 utcmma with A and B from smem and C stored in tmem, scale denotes if we +// want to accumulate C or not Something like: Accumulate or overwrite C. 1: +// read C, 0: ignore C [clear accumulators] +inline __device__ void +utcmma_asmem_bsmem_h_cta1(uint64_t sdescA, uint64_t sdescB, uint32_t tmemC, + uint64_t idescE, uint32_t scaleC) { + uint32_t mask[4] = {0, 0, 0, 0}; // TODO : Figure out what this mask means, + // CUTLASS hardcodes to all zeros + asm volatile("{\n\t" + ".reg .pred p;\n\t" + "setp.ne.b32 p, %4, 0;\n\t" + "tcgen05.mma.cta_group::1.kind::f16 [%0], %1, %2, %3, {%5, %6, " + "%7, %8}, p; \n\t" + "}\n" + : + : "r"(tmemC), "l"(sdescA), "l"(sdescB), + "r"(uint32_t(idescE >> 32)), "r"(scaleC), "r"(mask[0]), + "r"(mask[1]), "r"(mask[2]), "r"(mask[3])); +} + +// cta1 utcmma with A from tmem and B from smem and C stored in tmem, scale +// denotes if we want to accumulate C or not Something like: Accumulate or +// overwrite C. 1: read C, 0: ignore C [clear accumulators] +inline __device__ void +utcmma_atmem_bsmem_h_cta1(uint32_t tmemA, uint64_t sdescB, uint32_t tmemC, + uint64_t idescE, uint32_t scaleC) { + uint32_t mask[4] = {0, 0, 0, 0}; // TODO : Figure out what this mask means, + // CUTLASS hardcodes to all zeros + asm volatile("{\n\t" + ".reg .pred p;\n\t" + "setp.ne.b32 p, %4, 0;\n\t" + "tcgen05.mma.cta_group::1.kind::f16 [%0], [%1], %2, %3, {%5, " + "%6, %7, %8}, p; \n\t" + "}\n" + : + : "r"(tmemC), "r"(tmemA), "l"(sdescB), + "r"(uint32_t(idescE >> 32)), "r"(scaleC), "r"(mask[0]), + "r"(mask[1]), "r"(mask[2]), "r"(mask[3])); +} + +// Check for the utcmma to finish +inline __device__ void umma_arrive(uint32_t bar_intptr) { + asm volatile("tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::" + "cluster.b64 [%0];" ::"r"(bar_intptr)); +} + +inline __device__ uint32_t cast_smem_ptr_to_uint(void const *const ptr) { + return static_cast(__cvta_generic_to_shared(ptr)); +} + +inline __device__ void fence_view_async_shared(void) { + asm volatile("fence.proxy.async.shared::cta;\n"); +} + +inline __device__ void sts_128(r32 dst, r32 src[4]) { + asm volatile("st.shared.v4.b32 [%0], {%1, %2, %3, %4};\n" ::"r"(dst), + "r"(src[0]), "r"(src[1]), "r"(src[2]), "r"(src[3])); +} + +inline __device__ void lds_128(r32 dst[4], r32 src) { + asm volatile("ld.shared.v4.b32 {%0, %1, %2, %3}, [%4];\n" + : "=r"(dst[0]), "=r"(dst[1]), "=r"(dst[2]), "=r"(dst[3]) + : "r"(src)); +} + +inline __device__ void lds_32(r32 dst[1], r32 src) { + asm volatile("ld.shared.b32 %0, [%1];\n" : "=r"(dst[0]) : "r"(src)); +} + +inline __device__ void stg_32(void *ptr, r32 val[1]) { + uint32_t *p = reinterpret_cast(ptr); + p[0] = val[0]; +} + +inline __device__ void stg_8(void *ptr, r32 val[1]) { + uint8_t *p = reinterpret_cast(ptr); + p[0] = reinterpret_cast(val[0]); +} + +inline __device__ void ldg_16(r32 dst[1], const void *ptr, bool pred = true) { + if (pred) { + *reinterpret_cast(dst) = + *reinterpret_cast(ptr); + } else { + *reinterpret_cast(dst) = 0; + } +} + +inline __device__ void ldg_8(r32 dst[1], const void *ptr, bool pred = true) { + if (pred) { + *reinterpret_cast(dst) = *reinterpret_cast(ptr); + } else { + *reinterpret_cast(dst) = 0; + } +} + +inline __device__ void ldg_16_reg(r32 dst[1], const void *ptr, + bool pred = true) { + if (pred) { + uint16_t tmp = *reinterpret_cast(ptr); + asm volatile("{\nmov.b32 %0, {%1, %1};\n}\n" : "=r"(dst[0]) : "h"(tmp)); + } else { + dst[0] = 0; + } +} + +inline __device__ void ldg_8_reg(r32 dst[1], const void *ptr, + bool pred = true) { + if (pred) { + uint8_t tmp_val = *reinterpret_cast(ptr); + uint8_t *tmp_ptr = reinterpret_cast(dst); + tmp_ptr[0] = tmp_ptr[1] = tmp_ptr[2] = tmp_ptr[3] = tmp_val; + } else { + dst[0] = 0; + } +} + +// Forward (non-ReLU) Activations +// Backward (non-ReLU) Activations +// Forward ReLU activations +// Relu helper function: clip the lower part +// Relu helper function: clip the upper part +// Forward ReLU: clip only lower part +// Forward ReLU: clip lower part and upper part +// Forward ReLU: clip only lower part, but add slope to lower clip +// Forward ReLU: clip lower part and upper part, but add slope to lower clip +#if __CUDA_ARCH__ >= 900 +#endif + +// optimization opportunity exists here +inline __device__ void fp32x2_to_fp16x2(r32 dst[1], const r32 src[2]) { + asm volatile("cvt.rn.f16x2.f32 %0, %1, %2;\n" + : "=r"(dst[0]) + : "r"(src[1]), "r"(src[0])); +} + +inline __device__ void fp32x2_to_fp16x2_relu(r32 dst[1], const r32 src[2]) { + asm volatile("cvt.rn.relu.f16x2.f32 %0, %1, %2;\n" + : "=r"(dst[0]) + : "r"(src[1]), "r"(src[0])); +} + +inline __device__ void fp32x2_to_bf16x2(r32 dst[1], const r32 src[2]) { + asm volatile("cvt.rn.bf16x2.f32 %0, %1, %2;\n" + : "=r"(dst[0]) + : "r"(src[1]), "r"(src[0])); +} + +// mma_pipeline_op 1 includes +typedef struct AttentionDescriptor { + // Input parameters + // b - batch + // q_h - num heads of q/dq/o/do + // k_h - num heads of k/dk + // v_h - num heads of v/dv + // s_q - max sequence length of q + // s_kv - max sequence length of kv + // d - hidden dim (head dim) + uint32_t b, q_h, k_h, v_h, s_q, s_kv, d_qk, d_v; + uint16_t q_heads_per_k, q_heads_per_v, min_q_heads_per_kv; +} AttentionDescriptor_t; + + +inline __device__ void fence_view_async_tmem_load() { + asm volatile("{\n\ttcgen05.wait::ld.sync.aligned; \n}" ::); +} + +inline __device__ void fence_view_async_tmem_store() { + asm volatile("{\n\ttcgen05.wait::st.sync.aligned; \n}" ::); +} + +inline __device__ void tmem_allocate_1sm(uint32_t num_columns, + uint32_t smem_addr) { + asm volatile( + "tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], %1;" ::"r"( + smem_addr), + "r"(num_columns)); +} + +inline __device__ void tmem_free_1sm(uint32_t num_columns, uint32_t tmem_addr) { + asm volatile( + "{\n\ttcgen05.dealloc.cta_group::1.sync.aligned.b32 %0, %1; \n\t}" ::"r"( + tmem_addr), + "r"(num_columns)); +} + +inline __device__ void tmem_release_allocation_lock_1sm() { +)KERNEL" + R"KERNEL( + asm volatile("tcgen05.relinquish_alloc_permit.cta_group::1.sync.aligned;" ::); +} + +///////////////////////////////// +/// TMEM STTM related instructions +///////////////////////////////// + +// 32 data path lanes, 32-bit pattern, repeated 128 times +// 32 data path lanes, 32-bit pattern, repeated 64 times +// 32 data path lanes, 32-bit pattern, repeated 32 times +inline __device__ void sttm_32dp32bit_x32(uint32_t dst, r32 src[32]) { + asm volatile("tcgen05.st.sync.aligned.32x32b.x32.b32" + "[%0]," + "{%1, %2, %3, %4," + "%5, %6, %7, %8," + "%9, %10, %11, %12," + "%13, %14, %15, %16," + "%17, %18, %19, %20," + "%21, %22, %23, %24," + "%25, %26, %27, %28," + "%29, %30, %31, %32};\n" + : + : "r"(dst), "r"(src[0]), "r"(src[1]), "r"(src[2]), "r"(src[3]), + "r"(src[4]), "r"(src[5]), "r"(src[6]), "r"(src[7]), + "r"(src[8]), "r"(src[9]), "r"(src[10]), "r"(src[11]), + "r"(src[12]), "r"(src[13]), "r"(src[14]), "r"(src[15]), + "r"(src[16]), "r"(src[17]), "r"(src[18]), "r"(src[19]), + "r"(src[20]), "r"(src[21]), "r"(src[22]), "r"(src[23]), + "r"(src[24]), "r"(src[25]), "r"(src[26]), "r"(src[27]), + "r"(src[28]), "r"(src[29]), "r"(src[30]), "r"(src[31])); +} + +// 32 data path lanes, 32-bit pattern, repeated 16 times +inline __device__ void sttm_32dp32bit_x16(uint32_t dst, r32 src[16]) { + asm volatile("tcgen05.st.sync.aligned.32x32b.x16.b32" + "[%0]," + "{%1, %2, %3, %4," + "%5, %6, %7, %8," + "%9, %10, %11, %12," + "%13, %14, %15, %16};\n" + : + : "r"(dst), "r"(src[0]), "r"(src[1]), "r"(src[2]), "r"(src[3]), + "r"(src[4]), "r"(src[5]), "r"(src[6]), "r"(src[7]), + "r"(src[8]), "r"(src[9]), "r"(src[10]), "r"(src[11]), + "r"(src[12]), "r"(src[13]), "r"(src[14]), "r"(src[15])); +} + +// 32 data path lanes, 32-bit pattern, repeated 8 times +inline __device__ void sttm_32dp32bit_x8(uint32_t dst, r32 src[8]) { + asm volatile("tcgen05.st.sync.aligned.32x32b.x8.b32" + "[%0]," + "{%1, %2, %3, %4," + "%5, %6, %7, %8};\n" + : + : "r"(dst), "r"(src[0]), "r"(src[1]), "r"(src[2]), "r"(src[3]), + "r"(src[4]), "r"(src[5]), "r"(src[6]), "r"(src[7])); +} + +// 32 data path lanes, 32-bit pattern, repeated 4 times +// 32 data path lanes, 32-bit pattern, repeated 2 times +inline __device__ void sttm_32dp32bit_x2(uint32_t dst, r32 src[2]) { + asm volatile("tcgen05.st.sync.aligned.32x32b.x2.b32" + "[%0]," + "{%1, %2};\n" + : + : "r"(dst), "r"(src[0]), "r"(src[1])); +} + +// 32 data path lanes, 32-bit pattern, repeated 1 times +inline __device__ void sttm_32dp32bit_x1(uint32_t dst, r32 src[1]) { + asm volatile("tcgen05.st.sync.aligned.32x32b.x1.b32" + "[%0]," + "{%1};\n" + : + : "r"(dst), "r"(src[0])); +} + +///////////////////////////////// +/// TMEM LDTM related instructions +///////////////////////////////// + +// 32 data path lanes, 32-bit pattern, repeated 128 times +// 32 data path lanes, 32-bit pattern, repeated 64 times +// 32 data path lanes, 32-bit pattern, repeated 32 times +inline __device__ void ldtm_32dp32bit_x32(r32 dst[32], uint32_t src_addr) { + asm volatile("tcgen05.ld.sync.aligned.32x32b.x32.b32" + "{%0, %1, %2, %3," + "%4, %5, %6, %7," + "%8, %9, %10, %11," + "%12, %13, %14, %15," + "%16, %17, %18, %19," + "%20, %21, %22, %23," + "%24, %25, %26, %27," + "%28, %29, %30, %31}," + "[%32];\n" + : "=r"(dst[0]), "=r"(dst[1]), "=r"(dst[2]), "=r"(dst[3]), + "=r"(dst[4]), "=r"(dst[5]), "=r"(dst[6]), "=r"(dst[7]), + "=r"(dst[8]), "=r"(dst[9]), "=r"(dst[10]), "=r"(dst[11]), + "=r"(dst[12]), "=r"(dst[13]), "=r"(dst[14]), "=r"(dst[15]), + "=r"(dst[16]), "=r"(dst[17]), "=r"(dst[18]), "=r"(dst[19]), + "=r"(dst[20]), "=r"(dst[21]), "=r"(dst[22]), "=r"(dst[23]), + "=r"(dst[24]), "=r"(dst[25]), "=r"(dst[26]), "=r"(dst[27]), + "=r"(dst[28]), "=r"(dst[29]), "=r"(dst[30]), "=r"(dst[31]) + : "r"(src_addr)); +} + +// 32 data path lanes, 32-bit pattern, repeated 16 times +// 32 data path lanes, 32-bit pattern, repeated 8 times +inline __device__ void ldtm_32dp32bit_x8(r32 dst[8], uint32_t src_addr) { + asm volatile("tcgen05.ld.sync.aligned.32x32b.x8.b32" + "{%0, %1, %2, %3," + "%4, %5, %6, %7}," + "[%8];\n" + : "=r"(dst[0]), "=r"(dst[1]), "=r"(dst[2]), "=r"(dst[3]), + "=r"(dst[4]), "=r"(dst[5]), "=r"(dst[6]), "=r"(dst[7]) + : "r"(src_addr)); +} + +// 32 data path lanes, 32-bit pattern, repeated 4 times +// 32 data path lanes, 32-bit pattern, repeated 2 times +inline __device__ void ldtm_32dp32bit_x2(r32 dst[2], uint32_t src_addr) { + asm volatile("tcgen05.ld.sync.aligned.32x32b.x2.b32" + "{%0, %1}," + "[%2];\n" + : "=r"(dst[0]), "=r"(dst[1]) + : "r"(src_addr)); +} + +// 32 data path lanes, 32-bit pattern, repeated 1 times +inline __device__ void ldtm_32dp32bit_x1(r32 dst[1], uint32_t src_addr) { + asm volatile("tcgen05.ld.sync.aligned.32x32b.x1.b32" + "{%0}," + "[%1];\n" + : "=r"(dst[0]) + : "r"(src_addr)); +} + +// CUtensorMap, when issued using bulk copy async functions, are cached in +// constant cache. But the producer kernel may have directly written to global +// memory, without invalidating this constant cache. This acquire fence, +// invalidates the memory address in constant cache, using UTMACCTL.IV. +inline __device__ void tma_descriptor_fence_acquire(CUtensorMap const *p_desc) { +#if (__CUDA_ARCH__ >= 900) && (CUDACC_VERSION >= 123) + uint64_t gmem_int_desc = reinterpret_cast(p_desc); + asm volatile("fence.proxy.tensormap::generic.acquire.gpu [%0], 128;\n" + : + : "l"(gmem_int_desc) + : "memory"); + asm volatile("cvta.global.u64 %0, %0;\n" + : + : "l"(gmem_int_desc), "l"(gmem_int_desc) + : "memory"); +#endif +} + +inline __device__ void +utmaldg_4d_tiled(const void *p_desc, + uint32_t urb0, // smem offset + uint32_t urb1, // smem barrier offset + int32_t urb2, // m + int32_t urb3, // n + int32_t urb4, // b + int32_t urb5, // extra coord + const uint32_t elect_one, + const uint64_t mem_desc = MEM_DESC_DEFAULT) { + if (elect_one) { +#if __CUDA_ARCH__ >= 1200 && __CUDA_ARCH__ < 1300 + asm volatile( + "cp.async.bulk.tensor.4d.shared::cta.global.mbarrier::complete_tx::" + "bytes.L2::cache_hint [%0], [%1, {%2, %3, %4, %5}], [%6], %7;\n" ::"r"( + urb0), + "l"(reinterpret_cast(p_desc)), "r"(urb2), "r"(urb3), + "r"(urb4), "r"(urb5), "r"(urb1), "l"(mem_desc) + : "memory"); +#else + asm volatile( + "cp.async.bulk.tensor.4d.shared::cluster.global.mbarrier::complete_tx::" + "bytes.L2::cache_hint [%0], [%1, {%2, %3, %4, %5}], [%6], %7;\n" ::"r"( + urb0), + "l"(reinterpret_cast(p_desc)), "r"(urb2), "r"(urb3), + "r"(urb4), "r"(urb5), "r"(urb1), "l"(mem_desc) + : "memory"); +#endif + } +} + +inline __device__ void +utmaldg_4d_tiled_multicast(const void *p_desc, + uint32_t urb0, // smem offset + uint32_t urb1, // smem barrier offset + int32_t urb2, // m + int32_t urb3, // n + int32_t urb4, // b + int32_t urb5, // extra coord + const uint16_t mcast_mask, const uint32_t elect_one, + const uint64_t mem_desc = MEM_DESC_DEFAULT) { + if (elect_one) { + asm volatile("cp.async.bulk.tensor.4d.shared::cluster.global.mbarrier::" + "complete_tx::bytes.multicast::cluster.L2::cache_hint [%0], " + "[%1, {%2, %3, %4, %5}], [%6], %7, %8;\n" ::"r"(urb0), + "l"(reinterpret_cast(p_desc)), "r"(urb2), "r"(urb3), + "r"(urb4), "r"(urb5), "r"(urb1), "h"(mcast_mask), "l"(mem_desc) + : "memory"); + } +} + +#if (__CUDA_ARCH__ >= 1000) && (CUDACC_VERSION >= 128) +inline __device__ void +utmaldg_4d_tiled_2cta(const void *p_desc, + uint32_t urb0, // smem offset + uint32_t urb1, // smem barrier offset + int32_t urb2, // m + int32_t urb3, // n + int32_t urb4, // b + int32_t urb5, // extra coord + const uint32_t elect_one, + const uint64_t mem_desc = MEM_DESC_DEFAULT) { + if (elect_one) { + asm volatile("cp.async.bulk.tensor.4d.cta_group::2.shared::cluster.global." + "mbarrier::complete_tx::bytes.L2::cache_hint [%0], [%1, {%2, " + "%3, %4, %5}], [%6], %7;\n" ::"r"(urb0), + "l"(reinterpret_cast(p_desc)), "r"(urb2), "r"(urb3), + "r"(urb4), "r"(urb5), "r"(urb1 & MMA_PEER_BIT_MASK), + "l"(mem_desc) + : "memory"); + } +} + +inline __device__ void +utmaldg_4d_tiled_multicast_2cta(const void *p_desc, + uint32_t urb0, // smem offset + uint32_t urb1, // smem barrier offset + int32_t urb2, // m + int32_t urb3, // n + int32_t urb4, // b + int32_t urb5, // extra coord + const uint16_t mcast_mask, + const uint32_t elect_one, + const uint64_t mem_desc = MEM_DESC_DEFAULT) { + if (elect_one) { + asm volatile( + "cp.async.bulk.tensor.4d.cta_group::2.shared::cluster.global.mbarrier::" + "complete_tx::bytes.multicast::cluster.L2::cache_hint [%0], [%1, {%2, " + "%3, %4, %5}], [%6], %7, %8;\n" ::"r"(urb0), + "l"(reinterpret_cast(p_desc)), "r"(urb2), "r"(urb3), + "r"(urb4), "r"(urb5), "r"(urb1 & MMA_PEER_BIT_MASK), "h"(mcast_mask), + "l"(mem_desc) + : "memory"); + } +} +#endif + +inline __device__ void +utmastg_atomicAdd_4d_tiled(const void *p_desc, uint32_t smem_offset, + int32_t urb0, // m + int32_t urb1, // n + int32_t urb2, // h + int32_t urb3, // b + const uint32_t elect_one, + uint64_t mem_desc = MEM_DESC_DEFAULT) { + if (elect_one) { + asm volatile( + "cp.reduce.async.bulk.tensor.4d.global.shared::cta.add.bulk_group [%0, " + "{%1, %2, %3, %4}], [%5];\n" ::"l"(reinterpret_cast(p_desc)), + "r"(urb0), "r"(urb1), "r"(urb2), "r"(urb3), "r"(smem_offset) + : "memory"); + } +} + +inline __device__ void utmastg_4d_tiled(const void *p_desc, + uint32_t smem_offset, + int32_t urb0, // m + int32_t urb1, // n + int32_t urb2, // h + int32_t urb3, // b + uint64_t mem_desc = MEM_DESC_DEFAULT) { + asm volatile( + "cp.async.bulk.tensor.4d.global.shared::cta.bulk_group [%0, {%1, %2, %3, " + "%4}], [%5];\n" ::"l"(reinterpret_cast(p_desc)), + "r"(urb0), "r"(urb1), "r"(urb2), "r"(urb3), "r"(smem_offset) + : "memory"); +} + +inline __device__ void utmastg_5d_tiled(const void *p_desc, + uint32_t smem_offset, + int32_t urb0, // m + int32_t urb1, // n + int32_t urb2, // h + int32_t urb3, // b + int32_t urb4, // lean_tile_id + uint64_t mem_desc = MEM_DESC_DEFAULT) { + asm volatile( + "cp.async.bulk.tensor.5d.global.shared::cta.bulk_group [%0, {%1, %2, %3, " + "%4, %5}], [%6];\n" ::"l"(reinterpret_cast(p_desc)), + "r"(urb0), "r"(urb1), "r"(urb2), "r"(urb3), "r"(urb4), "r"(smem_offset) + : "memory"); +} + +// mma_loop_op 2 includes + +// global_load_shared_store_op 4 includes + +// shared_load_op 5 includes + +// shared_load_op 6 includes + +// global_load_shared_store_op 7 includes + +// mma_op 8 includes + +// global_load_op 10 includes + +// pointwise_calc_op 9 includes + +// softmax_op 12 includes + +// global_store_op 13 includes + +// global_store_op 14 includes + +// shared_load_op 15 includes + +// global_load_shared_store_op 16 includes + +// mma_op 17 includes + +// global_store_op 20 includes + +// output_loop_op 3 includes + +// shared_store_op 18 includes + +// shared_load_op 19 includes +} // namespace fort +)KERNEL" + R"KERNEL( + +// receive_op 0 types +using namespace fort; + +static constexpr int THREADS_PER_WARP_0 = 32; +static constexpr int WARPS_PER_GROUP_0 = FORT_MIN(4, 512 / THREADS_PER_WARP_0); +static constexpr int THREADS_PER_WARP_GROUP_0 = + THREADS_PER_WARP_0 * WARPS_PER_GROUP_0; +static constexpr int BITS_PER_REGISTER_0 = 32; // NOTE: sizeof(uint) +static constexpr int BYTES_PER_REGISTER_0 = BITS_PER_REGISTER_0 / 8; +static constexpr int THREADS_PER_GROUP_0 = + WARPS_PER_GROUP_0 * THREADS_PER_WARP_0; + +// mma_pipeline_op 1 types +static constexpr int BYTES_PER_BANK_1 = 16; +static constexpr int BITS_PER_ELEMENT_1 = 16; +static constexpr int ELEMS_PER_BANK_1 = + BYTES_PER_BANK_1 * 8 / BITS_PER_ELEMENT_1; +static constexpr int BYTES_PER_ELEMENT_1 = BITS_PER_ELEMENT_1 / 8; +static constexpr int BYTES_PER_FP16_1 = 2; +static constexpr int ELEMS_PER_VECTOR_1 = + BYTES_PER_REGISTER_0 / BYTES_PER_ELEMENT_1; +static constexpr int BITS_PER_ACC_1 = 32; +static constexpr int BYTES_PER_ACC_1 = BITS_PER_ACC_1 / 8; +static constexpr int TILE_M_1 = 128; +static constexpr int TILE_N_1 = 128; +static constexpr int TILE_K_1 = 128; +static constexpr int TILE_O_1 = FORT_MIN(TILE_K_1, 128); +static constexpr int TILE_V_1 = TILE_K_1; +static constexpr int TILE_DV_1 = FORT_MIN(TILE_K_1, 128); +static constexpr int WARP_TILE_M_1 = 16; +static constexpr int WARP_TILE_N_1 = 16; +static constexpr int WARP_TILE_K_1 = 32 / BYTES_PER_ELEMENT_1; +static constexpr int WARPS_M_1 = 64 / 16; +static constexpr int WARPS_N_1 = 16 / 16; +static constexpr int WARPS_K_1 = 16 / 16; +static constexpr int STAGES_1 = 1; +static constexpr int GROUPS_M_1 = 1; +static constexpr int GMMA_TILE_M_1 = WARPS_M_1 * WARP_TILE_M_1; +static constexpr int GMMA_TILE_K_1 = WARPS_K_1 * WARP_TILE_K_1; +static constexpr int WARP_TILES_M_1 = TILE_M_1 / WARP_TILE_M_1; +static constexpr int WARP_TILES_N_1 = TILE_N_1 / WARP_TILE_N_1; +static constexpr int WARP_TILES_K_1 = TILE_K_1 / WARP_TILE_K_1; +static constexpr int WARP_REGS_1 = TILE_N_1; +static constexpr int REGS_M_1 = WARP_TILES_M_1 / WARPS_M_1; +static constexpr int REGS_N_1 = WARP_TILES_N_1 / WARPS_N_1; +static constexpr int REGS_K_1 = WARP_TILES_K_1 / WARPS_K_1; +static constexpr int REGS_K_FOR_DP_1 = TILE_V_1 / 16; +static constexpr int REGS_dQ_1 = TILE_O_1 / 16; +static constexpr int REGS_O_1 = REGS_K_1 * 1; + +static constexpr int CGA_M_1 = 1; +static constexpr int CGA_N_1 = 1; +static constexpr int CTA_MMA_1 = 1; +static constexpr int M_TILES_PER_OUTPUT_TILE_1 = 4; +static constexpr int N_TILES_PER_OUTPUT_TILE_1 = 1; + +static constexpr int CTA_TILE_M_1 = TILE_M_1; +static constexpr int CTA_TILE_N_1 = 128; +static constexpr int CTA_TILE_K_1 = 128; + +static constexpr float ln2 = 0.6931471805599453094f; +static constexpr float inv_ln2 = 1.4426950408889634074f; + +static constexpr uint32_t SchedulerPipelineStageCount = 1; +static constexpr uint32_t AccumulatorPipelineStageCount = 1; +static constexpr uint32_t SmemPipelineStageCount = 1; +static constexpr uint32_t num_columns_per_tmem = 512; +static constexpr uint32_t columns_per_allocation_slice = 32; +static constexpr int stages_q = 2; +static constexpr int stages_kv = 2 * CTA_MMA_1; +static constexpr int Tiles_Q = 2; + +static constexpr int SoftmaxWarpGroups = 2; +static constexpr int CorrectionWarpGroups = 1; +static constexpr int SoftmaxWarps = SoftmaxWarpGroups * 4; +static constexpr int CorrectionWarps = 4; + +static constexpr int MmaWarpRegs = 40; // warp 12 +static constexpr int TmaldgWarpRegs = 40; // warp 13 +static constexpr int TmastgWarpRegs = 40; // warp 14 +static constexpr int SchedulerWarpRegs = 40; // warp 15 +static constexpr int CorrectionWarpRegs = 88; // warps 8-11 +static constexpr int SoftmaxWarpRegs = 192; // warps 0-7 + +static constexpr int threads_per_warp = 32; +static constexpr int warps_per_group = 4; +static constexpr int threads_per_group = threads_per_warp * warps_per_group; +static constexpr int threads_per_cta = + threads_per_warp * (warps_per_group + SoftmaxWarps + CorrectionWarps); + +static constexpr int BUFFERS_Q = stages_q; +static constexpr int BUFFERS_K = stages_kv; +static constexpr int BUFFERS_V = stages_kv; +static constexpr int BUFFERS_S = stages_kv; + +using ElementQ = fort::bfloat16_t; +using ElementK = fort::bfloat16_t; +using ElementV = fort::bfloat16_t; +using ElementS = fort::bfloat16_t; +using ElementO = fort::bfloat16_t; +using ElementAccumulator = float; + +static constexpr int BYTES_PER_ELEMENT = 2; +static constexpr int BYTES_PER_ACC = 4; + +static constexpr int BMM1_TILE_M = TILE_M_1; +static constexpr int BMM1_TILE_N = 128; +static constexpr int BMM1_TILE_K = 128; + +static constexpr int BMM2_TILE_M = TILE_M_1; +static constexpr int BMM2_TILE_N = 128; +static constexpr int BMM2_TILE_K = 128; + +static constexpr uint32_t qBufferElems = BMM1_TILE_M * BMM1_TILE_K; +static constexpr uint32_t kBufferElems = BMM1_TILE_N * BMM1_TILE_K / CTA_MMA_1; +static constexpr uint32_t vBufferElems = BMM2_TILE_N * BMM2_TILE_K / CTA_MMA_1; +static constexpr uint32_t oBufferElems = + FORT_MAX(8, BMM2_TILE_M) * (BMM2_TILE_N + 0); +static constexpr uint32_t sBufferElems = FORT_MAX(8, BMM1_TILE_M) * BMM1_TILE_N; + +static constexpr uint32_t qTmaTransactionBytes = + qBufferElems * sizeof(ElementQ) * CTA_MMA_1; +static constexpr uint32_t kTmaTransactionBytes = + kBufferElems * sizeof(ElementK) * CTA_MMA_1; +static constexpr uint32_t vTmaTransactionBytes = + vBufferElems * sizeof(ElementV) * CTA_MMA_1; + +static constexpr int UTCMMA_TILE_M = 128; +static constexpr int UTCMMA_TILE_N = 128; +static constexpr int UTCMMA_TILE_K = 32 / BYTES_PER_ELEMENT; + +static constexpr int BMM1_XMMAS_K = BMM1_TILE_K / UTCMMA_TILE_K; +static constexpr int BMM2_XMMAS_K = BMM2_TILE_K / UTCMMA_TILE_K; +static constexpr int BYTES_PER_MMA_K = UTCMMA_TILE_K * BYTES_PER_ELEMENT; + +static constexpr int SMEM_BUFFER_SIZE_Q = + BMM1_TILE_M * BMM1_TILE_K * BYTES_PER_ELEMENT; +static constexpr int SMEM_BUFFER_SIZE_K = + BMM1_TILE_N * BMM1_TILE_K * BYTES_PER_ELEMENT / CTA_MMA_1; +static constexpr int SMEM_BUFFER_SIZE_V = + BMM2_TILE_N * BMM2_TILE_K * BYTES_PER_ELEMENT / CTA_MMA_1; +static constexpr int SMEM_BUFFER_SIZE_S = + FORT_MAX(8, BMM1_TILE_M) * BMM1_TILE_N * BYTES_PER_ELEMENT; + +static constexpr int SOFTMAX_BARRIER = 1; +static constexpr int BAND_BIAS_BARRIER = 5; +static constexpr int TMEM_ALLOC_SOFTMAX_BARRIER = 6; +static constexpr int TMEM_ALLOC_CORRECTION_BARRIER = 7; + +struct SharedStorage { + alignas(1024) ElementQ smem_Q[qBufferElems * stages_q]; + alignas(1024) ElementK smem_K[kBufferElems * stages_kv]; + alignas(1024) ElementV smem_V[vBufferElems * stages_kv]; + alignas(1024) ElementO smem_O[oBufferElems]; + + alignas(16) uint64_t tma_o_0_empty_mbar[1]; + + alignas(16) uint64_t tma_k_empty_mbar[stages_kv]; + alignas(16) uint64_t tma_k_full_mbar[stages_kv]; + + alignas(16) uint64_t tma_v_empty_mbar[stages_kv]; + alignas(16) uint64_t tma_v_full_mbar[stages_kv]; + + alignas(16) uint64_t tma_q_empty_mbar[stages_q]; + alignas(16) uint64_t tma_q_full_mbar[stages_q]; + + alignas(16) uint64_t bmm2_tile0_done_mbar[1]; + alignas(16) uint64_t bmm2_tile0_ready_mbar[2]; + + alignas(16) uint64_t stat_tile0_full_mbar[1]; + alignas(16) uint64_t stat_tile0_empty_mbar[1]; + alignas(16) uint64_t final_stat_tile0_empty_mbar[1]; + alignas(16) uint64_t + tma_o_0_full_mbar[FORT_DIV_UP(TILE_O_1 * sizeof(ElementO), 128)]; + alignas(16) uint64_t + tma_o_1_full_mbar[FORT_DIV_UP(TILE_O_1 * sizeof(ElementO), 128)]; + + alignas(16) uint64_t epilogue_done_mbar[1]; + alignas(16) uint64_t tmaldg_tile_started_mbar[1]; + + alignas(16) uint64_t bmm1_tile0_done_mbar[1]; + alignas(16) uint64_t bmm1_tile1_done_mbar[1]; + + alignas(16) uint64_t bmm2_tile1_done_mbar[1]; + alignas(16) uint64_t bmm2_tile1_ready_mbar[2]; + + alignas(16) uint64_t stat_tile1_full_mbar[1]; + alignas(16) uint64_t stat_tile1_empty_mbar[1]; + alignas(16) uint64_t final_stat_tile1_empty_mbar[1]; + + alignas(16) uint64_t empty_mainloop_mbar[1]; + + alignas(16) uint64_t scheduler_mbar[SchedulerPipelineStageCount]; + alignas(16) uint64_t read_tile_id_done_mbar[SchedulerPipelineStageCount]; + alignas(16) uint64_t tmem_dealloc_smem_bar[1]; + + uint32_t tmem_base_ptrs; + + alignas(16) uint32_t tile_id[8 * SchedulerPipelineStageCount]; +}; + +// global_load_shared_store_op 4 types +static constexpr int BITS_PER_ELEMENT_4 = 16; +static constexpr int BYTES_PER_ELEMENT_4 = BITS_PER_ELEMENT_4 / 8; +static constexpr int BYTES_PER_ACCESS_4 = 16; +static constexpr int TILE_M_4 = 128; +static constexpr int TILE_N_4 = 128; +static constexpr int BYTES_PER_SMEM_4 = + TILE_M_4 * TILE_N_4 * BYTES_PER_ELEMENT_4; + +// shared_load_op 5 types +static constexpr int BITS_PER_ELEMENT_5 = 16; +static constexpr int BYTES_PER_ELEMENT_5 = 2; +static constexpr int BYTES_PER_QUAD_5 = 16; +static constexpr int WARP_TILE_M_5 = 16; +static constexpr int WARP_TILE_N_5 = 16; +static constexpr int TILE_M_5 = 128; +static constexpr int TILE_N_5 = 128; +static constexpr int BYTES_PER_SMEM_5 = + TILE_M_5 * TILE_N_5 * BYTES_PER_ELEMENT_5; +static constexpr int WARPS_M_5 = 4; +static constexpr int WARPS_N_5 = 1; +static constexpr int WARP_TILES_M_5 = TILE_M_5 / WARP_TILE_M_5; +static constexpr int WARP_TILES_N_5 = TILE_N_5 / WARP_TILE_N_5; +static constexpr int THREADS_PER_WARP_TILE_M_5 = 8; +static constexpr int THREADS_PER_WARP_TILE_N_5 = 4; +static constexpr int REGS_M_5 = WARP_TILES_M_5 / WARPS_M_5; +static constexpr int REGS_N_5 = WARP_TILES_N_5 / WARPS_N_5; +static constexpr int BYTES_PER_LD_5 = 128; +static constexpr int SWIZZLE_SCALE_5 = + FORT_MIN(BYTES_PER_LD_5 / 16, WARPS_PER_GROUP_0 * 4); + +// shared_load_op 6 types +static constexpr int BITS_PER_ELEMENT_6 = 16; +static constexpr int BYTES_PER_ELEMENT_6 = 2; +static constexpr int BYTES_PER_QUAD_6 = 16; +static constexpr int WARP_TILE_M_6 = 16; +static constexpr int WARP_TILE_N_6 = 16; +static constexpr int TILE_M_6 = 128; +static constexpr int TILE_N_6 = 128; +static constexpr int BYTES_PER_SMEM_6 = + TILE_M_6 * TILE_N_6 * BYTES_PER_ELEMENT_6; +static constexpr int WARPS_M_6 = 1; +static constexpr int WARPS_N_6 = 1; +static constexpr int WARP_TILES_M_6 = TILE_M_6 / WARP_TILE_M_6; +static constexpr int WARP_TILES_N_6 = TILE_N_6 / WARP_TILE_N_6; +static constexpr int THREADS_PER_WARP_TILE_M_6 = 8; +static constexpr int THREADS_PER_WARP_TILE_N_6 = 4; +static constexpr int REGS_M_6 = WARP_TILES_M_6 / WARPS_M_6; +static constexpr int REGS_N_6 = WARP_TILES_N_6 / WARPS_N_6; +static constexpr int BYTES_PER_LD_6 = 128; +static constexpr int SWIZZLE_SCALE_6 = + FORT_MIN(BYTES_PER_LD_6 / 16, WARPS_PER_GROUP_0 * 4); + +// global_load_shared_store_op 7 types +static constexpr int BITS_PER_ELEMENT_7 = 16; +static constexpr int BYTES_PER_ELEMENT_7 = BITS_PER_ELEMENT_7 / 8; +static constexpr int BYTES_PER_ACCESS_7 = 16; +static constexpr int TILE_M_7 = 128; +static constexpr int TILE_N_7 = 128; +static constexpr int BYTES_PER_SMEM_7 = + TILE_M_7 * TILE_N_7 * BYTES_PER_ELEMENT_7; + +// mma_op 8 types +static constexpr int ROWS_PER_CORE_MATRIX_A_8 = 8; +static constexpr int COLS_PER_CORE_MATRIX_A_8 = 8; +static constexpr int ROWS_PER_CORE_MATRIX_B_8 = 8; +static constexpr int COLS_PER_CORE_MATRIX_B_8 = 8; + +// global_load_op 10 types +static constexpr int BITS_PER_ELEMENT_10 = 32; +static constexpr int BYTES_PER_ELEMENT_10 = 4; +static constexpr int REGISTERS_PER_VECTOR_10 = 1; +static constexpr int REGISTERS_PER_ACCESS_10 = 1; + +// pointwise_calc_op 9 types +static constexpr int WARP_TILE_M_9 = 16; +static constexpr int WARP_TILE_N_9 = 16; +static constexpr int TILE_M_9 = 128; +static constexpr int TILE_N_9 = 128; +static constexpr int WARPS_M_9 = 4; +static constexpr int WARPS_N_9 = 1; +static constexpr int WARP_TILES_M_9 = TILE_M_9 / WARP_TILE_M_9; +static constexpr int WARP_TILES_N_9 = TILE_N_9 / WARP_TILE_N_9; +static constexpr int WARP_REGS_9 = WARP_REGS_1; +static constexpr int REGS_M_9 = WARP_TILES_M_9 / WARPS_M_9; +static constexpr int REGS_N_9 = WARP_TILES_N_9 / WARPS_N_9; + +// mha_mask_op 11 types +static constexpr int WARP_TILE_M_11 = 16; +static constexpr int WARP_TILE_N_11 = 16; +static constexpr int TILE_M_11 = 128; +static constexpr int TILE_N_11 = 128; +static constexpr int WARPS_M_11 = 4; +static constexpr int WARPS_N_11 = 1; +static constexpr int WARP_TILES_M_11 = TILE_M_11 / WARP_TILE_M_11; +static constexpr int WARP_TILES_N_11 = TILE_N_11 / WARP_TILE_N_11; +static constexpr int WARP_REGS_11 = 8; +static constexpr int REGS_M_11 = WARP_TILES_M_11 / WARPS_M_11; +static constexpr int REGS_N_11 = WARP_TILES_N_11 / WARPS_N_11; +inline __device__ bool compute_diagonal_band_mask_11( +)KERNEL" + R"KERNEL( + const int row, const int col, const int actual_seqlen_kv, + const int actual_seqlen_q, const int shift_right_bound, + const int left_bound) { + constexpr bool is_bottom_right_alignment = false; + constexpr bool is_shift_right_bound = false; + constexpr bool is_right_bound = true; + constexpr bool is_left_bound = false; + constexpr bool need_oob_check = false; + + const int diag = is_bottom_right_alignment + ? row + (actual_seqlen_kv - actual_seqlen_q) + : row; + const int shifted_right_bound = + is_shift_right_bound ? diag + shift_right_bound : diag; + const bool right_bound_mask = + is_right_bound ? col <= shifted_right_bound : true; + const bool left_bound_mask = is_left_bound ? col + left_bound > diag : true; + const bool oob_check = + need_oob_check ? (row < actual_seqlen_q && col < actual_seqlen_kv) : true; + return right_bound_mask && left_bound_mask && oob_check; +} + +// softmax_op 12 types +static constexpr int BYTES_PER_ELEMENT_12 = 2; +static constexpr int WARP_TILE_M_12 = 16; +static constexpr int WARP_TILE_N_12 = 16; +static constexpr int TILE_M_12 = 128; +static constexpr int TILE_N_12 = 128; +static constexpr int WARPS_M_12 = 4; +static constexpr int WARPS_N_12 = 1; +static constexpr int WARP_TILES_M_12 = TILE_M_12 / WARP_TILE_M_12; +static constexpr int WARP_TILES_N_12 = TILE_N_12 / WARP_TILE_N_12; +static constexpr int THREADS_PER_WARP_TILE_M_12 = 8; +static constexpr int THREADS_PER_WARP_TILE_N_12 = 4; +static constexpr int WARP_REGS_12 = 8; +static constexpr int REGS_M_12 = WARP_TILES_M_12 / WARPS_M_12; +static constexpr int REGS_N_12 = WARP_TILES_N_12 / WARPS_N_12; +static constexpr int ROWS_PER_THREAD_12 = + TILE_M_12 / WARPS_M_12 / THREADS_PER_WARP_TILE_M_12; + +// global_store_op 13 types +static constexpr int BYTES_PER_ELEMENT_13 = 4; +static constexpr int WARP_TILE_M_13 = 16; +static constexpr int WARP_TILE_N_13 = 16; +static constexpr int TILE_M_13 = 128; +static constexpr int TILE_N_13 = 128; +static constexpr int WARPS_M_13 = 4; +static constexpr int WARPS_N_13 = 1; +static constexpr int WARP_TILES_M_13 = TILE_M_13 / WARP_TILE_M_13; +static constexpr int WARP_TILES_N_13 = TILE_N_13 / WARP_TILE_N_13; +static constexpr int THREADS_PER_WARP_TILE_M_13 = 8; +static constexpr int THREADS_PER_WARP_TILE_N_13 = 4; +static constexpr int ROWS_PER_THREAD_13 = + TILE_M_13 / WARPS_M_13 / THREADS_PER_WARP_TILE_M_13; + +// global_store_op 14 types +static constexpr int BYTES_PER_ELEMENT_14 = 4; +static constexpr int WARP_TILE_M_14 = 16; +static constexpr int WARP_TILE_N_14 = 16; +static constexpr int TILE_M_14 = 128; +static constexpr int TILE_N_14 = 128; +static constexpr int WARPS_M_14 = 4; +static constexpr int WARPS_N_14 = 1; +static constexpr int WARP_TILES_M_14 = TILE_M_14 / WARP_TILE_M_14; +static constexpr int WARP_TILES_N_14 = TILE_N_14 / WARP_TILE_N_14; +static constexpr int THREADS_PER_WARP_TILE_M_14 = 8; +static constexpr int THREADS_PER_WARP_TILE_N_14 = 4; +static constexpr int ROWS_PER_THREAD_14 = + TILE_M_14 / WARPS_M_14 / THREADS_PER_WARP_TILE_M_14; + +// shared_load_op 15 types +static constexpr int BITS_PER_ELEMENT_15 = 16; +static constexpr int BYTES_PER_ELEMENT_15 = 2; +static constexpr int BYTES_PER_QUAD_15 = 16; +static constexpr int WARP_TILE_M_15 = 16; +static constexpr int WARP_TILE_N_15 = 16; +static constexpr int TILE_M_15 = 128; +static constexpr int TILE_N_15 = 128; +static constexpr int BYTES_PER_SMEM_15 = + TILE_M_15 * TILE_N_15 * BYTES_PER_ELEMENT_15; +static constexpr int WARPS_M_15 = 1; +static constexpr int WARPS_N_15 = 1; +static constexpr int WARP_TILES_M_15 = TILE_M_15 / WARP_TILE_M_15; +static constexpr int WARP_TILES_N_15 = TILE_N_15 / WARP_TILE_N_15; +static constexpr int THREADS_PER_WARP_TILE_M_15 = 8; +static constexpr int THREADS_PER_WARP_TILE_N_15 = 4; +static constexpr int REGS_M_15 = WARP_TILES_M_15 / WARPS_M_15; +static constexpr int REGS_N_15 = WARP_TILES_N_15 / WARPS_N_15; +static constexpr int BYTES_PER_LD_15 = 128; +static constexpr int SWIZZLE_SCALE_15 = + FORT_MIN(BYTES_PER_LD_15 / 16, WARPS_PER_GROUP_0 * 4); + +// global_load_shared_store_op 16 types +static constexpr int BITS_PER_ELEMENT_16 = 16; +static constexpr int BYTES_PER_ELEMENT_16 = BITS_PER_ELEMENT_16 / 8; +static constexpr int BYTES_PER_ACCESS_16 = 16; +static constexpr int TILE_M_16 = 128; +static constexpr int TILE_N_16 = 128; +static constexpr int BYTES_PER_SMEM_16 = + TILE_M_16 * TILE_N_16 * BYTES_PER_ELEMENT_16; + +// mma_op 17 types +static constexpr int ROWS_PER_CORE_MATRIX_A_17 = 8; +static constexpr int COLS_PER_CORE_MATRIX_A_17 = 8; +static constexpr int ROWS_PER_CORE_MATRIX_B_17 = 8; +static constexpr int COLS_PER_CORE_MATRIX_B_17 = 8; + +// global_store_op 20 types + +// mma_loop_op 2 types +inline __device__ int2 compute_kv_loop_bounds( + const int row_coord, const int ROW_TILE_SIZE, const int COL_TILE_SIZE, + const int actual_seqlen_kv, const int actual_seqlen_q, + const int shift_right_bound, const int left_bound) { + constexpr bool is_right_bound = true; + constexpr bool is_right_bound_bottom_right_alignment = false; + constexpr bool is_shift_right_bound = false; + constexpr bool is_left_bound = false; + constexpr bool is_left_bound_bottom_right_alignment = false; + + const int right_bound_diagonal = + is_right_bound_bottom_right_alignment + ? row_coord + (actual_seqlen_kv - actual_seqlen_q) + : row_coord; + const int shifted_right_bound_diagonal = + is_shift_right_bound ? right_bound_diagonal + shift_right_bound + : right_bound_diagonal; + const int left_bound_diagonal = + is_left_bound_bottom_right_alignment + ? row_coord + (actual_seqlen_kv - actual_seqlen_q) + : row_coord; + + const int kv_loop_left_bound = + is_left_bound + ? FORT_MAX(0, (left_bound_diagonal - left_bound) / COL_TILE_SIZE) + : 0; + const int kv_loop_right_bound = + is_right_bound + ? FORT_MIN(FORT_DIV_UP(shifted_right_bound_diagonal + ROW_TILE_SIZE, + COL_TILE_SIZE), + FORT_DIV_UP(actual_seqlen_kv, COL_TILE_SIZE)) + : FORT_DIV_UP(actual_seqlen_kv, COL_TILE_SIZE); + return make_int2(kv_loop_left_bound, kv_loop_right_bound); +} + +// output_loop_op 3 types +static constexpr int BYTES_PER_BANK_3 = 16; +static constexpr int ELEMENTS_PER_VECTOR_3 = + FORT_MIN(64, CTA_TILE_N_1 / N_TILES_PER_OUTPUT_TILE_1); + +// shared_store_op 18 types + +// shared_load_op 19 types +static constexpr int BITS_PER_ELEMENT_19 = 32 * 1; +static constexpr int BYTES_PER_ELEMENT_19 = BITS_PER_ELEMENT_19 / 8; +static constexpr int BITS_PER_VECTOR_19 = + BITS_PER_ELEMENT_19 * ELEMENTS_PER_VECTOR_3; +static constexpr int BYTES_PER_VECTOR_19 = BITS_PER_VECTOR_19 / 8; +static constexpr int REGISTERS_PER_VECTOR_19 = + BITS_PER_VECTOR_19 / BITS_PER_REGISTER_0; + +// receive_op 0 code + +extern "C" __global__ +__launch_bounds__(512, 1) void cudnn_generated_oss_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_cga1x1x1_kernel0_0( + const AttentionDescriptor_t attnDesc, const FastDivisor_t num_head_divmod, + const FastDivisor_t l2_minor_divmod, const FastDivisor_t l2_major_divmod, + const FastDivisor_t l2_minor_residual_divmod, const int num_hb_quotient, + const int num_block, const int total_blocks, + __grid_constant__ const CUtensorMap tma_tensor_3, + __grid_constant__ const CUtensorMap tma_tensor_2, float tensor_4, + float tensor_13, void *tensor_7, fort::tensor_descriptor desc_7, + void *tensor_6, fort::tensor_descriptor desc_6, + __grid_constant__ const CUtensorMap tma_tensor_1, + __grid_constant__ const CUtensorMap tma_tensor_5) { + asm volatile(".pragma \"global knob ForceLateCommoning=1\";\n" ::: "memory"); + asm volatile(".pragma \"global knob HoistLate=3\";\n" ::: "memory"); + + extern __shared__ char smem_[]; + SharedStorage &shared_storage = *reinterpret_cast(smem_); + + // receive_op 0 decls + + // mma_pipeline_op 1 decls + const int tid = threadIdx.x % threads_per_group; + const int wid = threadIdx.x / 32; + const int tiw = threadIdx.x % 32; + + const uint32_t elect_one = elect_one_sync(); + + if (wid == 0 && elect_one) { + smem_bar_init(cast_smem_ptr_to_uint(&shared_storage.tma_o_0_empty_mbar[0]), + /* 1 tmastg */ 32); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.tmem_dealloc_smem_bar[0]), + /* 4 epilogue */ warps_per_group * 32 * CTA_MMA_1); + +#pragma unroll + for (int q_stage = 0; q_stage < stages_q; q_stage++) { + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.tma_q_empty_mbar[q_stage]), + /* 1 mma */ 1); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.tma_q_full_mbar[q_stage]), + /* 1 tma */ 1); + } + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.bmm2_tile0_ready_mbar[0]), + /* 4 Softmax */ warps_per_group * 32 * 2 * CTA_MMA_1); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.bmm2_tile0_done_mbar[0]), + /* 1 mma */ 1); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.stat_tile0_full_mbar[0]), + /* 4 Softmax */ warps_per_group * 32); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.stat_tile0_empty_mbar[0]), + /* 4 Softmax */ warps_per_group * 32); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.final_stat_tile0_empty_mbar[0]), + /* 4 Correction */ warps_per_group * 32 * CTA_MMA_1); + +#pragma unroll + for (int i = 0; i < TILE_O_1 * sizeof(ElementO); i += 128) { + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.tma_o_0_full_mbar[i / 128]), + /* 1 correction */ warps_per_group * 32); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.tma_o_1_full_mbar[i / 128]), + /* 1 correction */ warps_per_group * 32); + } + smem_bar_init(cast_smem_ptr_to_uint(&shared_storage.epilogue_done_mbar[0]), + /* 1 tmastg */ 32); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.tmaldg_tile_started_mbar[0]), 32); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.bmm2_tile1_ready_mbar[0]), + /* 4 Softmax */ warps_per_group * 32 * 2 * CTA_MMA_1); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.bmm2_tile0_ready_mbar[1]), + /* 4 Softmax */ warps_per_group * 32 * 1 * CTA_MMA_1); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.bmm2_tile1_ready_mbar[1]), + /* 4 Softmax */ warps_per_group * 32 * 1 * CTA_MMA_1); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.bmm1_tile0_done_mbar[0]), + /* 1 mma */ 1); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.bmm1_tile1_done_mbar[0]), + /* 1 mma */ 1); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.bmm2_tile1_done_mbar[0]), + /* 1 mma */ 1); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.stat_tile1_full_mbar[0]), + /* 4 Softmax */ warps_per_group * 32); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.stat_tile1_empty_mbar[0]), + /* 4 Softmax */ warps_per_group * 32); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.final_stat_tile1_empty_mbar[0]), + /* 4 Correction */ warps_per_group * 32 * CTA_MMA_1); + + smem_bar_init(cast_smem_ptr_to_uint(&shared_storage.empty_mainloop_mbar[0]), + /* 4 Softmax */ warps_per_group * 32 * 1 * CTA_MMA_1); + +#pragma unroll + for (int sched_steps = 0; sched_steps < SchedulerPipelineStageCount; + sched_steps++) { + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.scheduler_mbar[sched_steps]), + /* 1 sched */ 1); + smem_bar_init(cast_smem_ptr_to_uint( + &shared_storage.read_tile_id_done_mbar[sched_steps]), + (SoftmaxWarps + CorrectionWarps + 1 + 1) * CGA_M_1 * + CGA_N_1 + + CGA_M_1 * CGA_N_1 / CTA_MMA_1); + } + +#pragma unroll + for (int kv_stages = 0; kv_stages < stages_kv; kv_stages++) { + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.tma_k_empty_mbar[kv_stages]), + /* 1 mma */ CGA_M_1 / CTA_MMA_1 + CGA_N_1 - 1); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.tma_k_full_mbar[kv_stages]), + /* 1 tma/32 ldgsts */ 1); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.tma_v_full_mbar[kv_stages]), + /* 1 tma/32 ldgsts */ 1); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.tma_v_empty_mbar[kv_stages]), + /* 1 mma */ CGA_M_1 / CTA_MMA_1 + CGA_N_1 - 1); + } + } + __syncthreads(); + +)KERNEL" + R"KERNEL( + // mma_pipeline_op 1 code + + // mma_loop_op 2 code + uint32_t tile_id_and_coord[8] = {0}; + + int actual_seqlen_kv_1 = attnDesc.s_kv; + int actual_seqlen_q_1 = attnDesc.s_q; + int q_row_coord = 0; + int kv_loop_right_bound, kv_loop_left_bound; + + const int left_bound = 0; + const int shift_right_bound = 0; + + uint32_t tile_idx = blockIdx.x; + uint32_t bidhb, l2_mod; + uint32_t block = 0, bidhb_residual = 0; + + fastDivMod(l2_major_divmod, tile_idx, bidhb, l2_mod); + if (bidhb < num_hb_quotient) { + { + fastDivMod(l2_minor_divmod, l2_mod, block, bidhb_residual); + } + } else { + { + fastDivMod(l2_minor_residual_divmod, l2_mod, block, bidhb_residual); + } + } + + int bidhb_actual = bidhb * l2_minor_divmod.val + bidhb_residual; + uint32_t batch_idx, head_idx; + fastDivMod(num_head_divmod, bidhb_actual, batch_idx, head_idx); + block = num_block - 1 - block; + tile_id_and_coord[0] = head_idx; + tile_id_and_coord[1] = batch_idx; + tile_id_and_coord[2] = block; + tile_id_and_coord[3] = 1; + + int head_coord_q_offset_1 = 0; + uint32_t head_coord_1, head_coord_k_1, head_coord_v_1; + uint32_t blocked_row_coord = tile_id_and_coord[2]; + uint32_t head_coord_from_grid_1 = tile_id_and_coord[0]; + uint32_t batch_coord_1 = tile_id_and_coord[1]; + uint32_t lean_tile_id = tile_id_and_coord[0]; + + uint32_t is_valid_tile = true; + + head_coord_1 = head_coord_from_grid_1; + head_coord_k_1 = head_coord_1 / attnDesc.q_heads_per_k; + head_coord_v_1 = head_coord_1 / attnDesc.q_heads_per_v; + + q_row_coord = (blocked_row_coord * Tiles_Q * BMM1_TILE_M); + int2 kv_loop_bounds = compute_kv_loop_bounds( + q_row_coord, BMM1_TILE_M * Tiles_Q * CGA_M_1, TILE_N_1, + actual_seqlen_kv_1, actual_seqlen_q_1, shift_right_bound, left_bound); + kv_loop_left_bound = kv_loop_bounds.x; + kv_loop_right_bound = kv_loop_bounds.y; + if (actual_seqlen_q_1 == 0 || q_row_coord >= actual_seqlen_q_1) { + kv_loop_right_bound = kv_loop_left_bound; + } + + PipelineState sched_state(0, 0); // index, state + + // thread_specialize_op 21 code + if (wid >= 0 && wid < SoftmaxWarps) { + // Softmax + reg_alloc(); + + const uint32_t softmax_gid = wid / 4; + const uint32_t softmax_gid_s_offset = softmax_gid * BMM1_TILE_N; + + uint32_t base_tmem_addr; + named_barrier_wait(TMEM_ALLOC_SOFTMAX_BARRIER, + threads_per_warp * (SoftmaxWarps + 1)); + lds_32(&base_tmem_addr, + cast_smem_ptr_to_uint(&shared_storage.tmem_base_ptrs)); + uint32_t tmem_fp32_S = base_tmem_addr + 0 + softmax_gid_s_offset; + uint32_t tmem_fp16_S = + base_tmem_addr + BMM1_TILE_N / 2 + softmax_gid_s_offset; + uint32_t tmem_Stats = base_tmem_addr + 0 + softmax_gid_s_offset; + + r32 reg_8_0[BMM1_TILE_N]; + r32 fp32_stats[2]; + + // Memory barrier states + uint32_t bmm_mbar_state = 0; + uint32_t stat_mbar_state = 0; + uint32_t epilogue_state = 0; + +// Persistent loop over tiles +#pragma unroll 1 + while (is_valid_tile) { + int p_row_1 = q_row_coord + softmax_gid * TILE_M_1 + tid; + + // global_load_op 10 decls + r32 reg_10_0[1]; + reg_10_0[0] = reinterpret_cast(tensor_4); + const int oob_M_10 = 1; + const int oob_N_10 = 1; + + // pointwise_calc_op 9 decls + + // mha_mask_op 11 decls + + // softmax_op 12 decls + r32 reg_12_0[32]; + float2 total_sum = {0.0f, 0.0f}; + float2 local_sum = {0.0f, 0.0f}; + float total_max = NEG_INFINITY; + float actual_total_max = NEG_INFINITY; + static constexpr float inv_ln2_12 = 1.4426950408889634074f; + float total_max_scale_12 = inv_ln2_12; + const float bmm_scale = reinterpret_cast(tensor_4); + total_max_scale_12 *= bmm_scale; + + uint64_t &bmm1_mbar = softmax_gid == 0 + ? shared_storage.bmm1_tile0_done_mbar[0] + : shared_storage.bmm1_tile1_done_mbar[0]; + uint64_t &stat_empty_mbar = softmax_gid == 0 + ? shared_storage.stat_tile0_empty_mbar[0] + : shared_storage.stat_tile1_empty_mbar[0]; + uint64_t *bmm2_mbar = softmax_gid == 0 + ? &(shared_storage.bmm2_tile0_ready_mbar[0]) + : &(shared_storage.bmm2_tile1_ready_mbar[0]); + uint64_t *stat_full_mbar = + softmax_gid == 0 ? &(shared_storage.stat_tile0_full_mbar[0]) + : &(shared_storage.stat_tile1_full_mbar[0]); + + epilogue_state ^= 1; + wait_barrier(cast_smem_ptr_to_uint(&shared_storage.epilogue_done_mbar[0]), + epilogue_state); + stat_mbar_state ^= 1; + wait_barrier(cast_smem_ptr_to_uint(&stat_empty_mbar), stat_mbar_state); + +#pragma unroll 1 + for (int kv_loop = kv_loop_left_bound; + kv_loop < + kv_loop_right_bound - + FORT_DIV_UP(BMM1_TILE_M * CGA_M_1 * Tiles_Q, BMM1_TILE_N); + kv_loop += 1) { + int p_col_1 = kv_loop * TILE_N_1; + { + + wait_barrier(cast_smem_ptr_to_uint(&bmm1_mbar), bmm_mbar_state); + + r32 stat_reg[BMM1_TILE_N / 64]; +#pragma unroll + for (int ldtm_step = 0; ldtm_step < BMM1_TILE_N / 32; ldtm_step++) { + ldtm_32dp32bit_x32(®_8_0[32 * ldtm_step], + tmem_fp32_S + ldtm_step * 32); + } + constexpr bool use_ldtm_stat_max = false; + + // global_load_op 10 code + + // softmax_op 12 code + float current_max = + row_max_reduction_128_elems(reg_8_0) * total_max_scale_12; + + // Update total max + float total_max_tmp = fmaxf(current_max, total_max); + current_max = total_max; + total_max = total_max_tmp; + + float2 bmm_scale_x_ln2_x2 = + make_float2(total_max_scale_12, total_max_scale_12); + float total_max_scaled = + (total_max == NEG_INFINITY) ? 0.0f : total_max; + float2 minus_scaled_max_x2 = + make_float2(-total_max_scaled, -total_max_scaled); + + static constexpr int kMulPipeCount = 14; // must be multiple of 2 + static constexpr int kSubtractPipeCount = 12; // must be multiple of 2 + static constexpr int kFmaPipeCount = 12; // must be multiple of 2 + static constexpr int kConvertPipeCount = 8; // must be multiple of 2 + static constexpr int kReleasePipeCount = 4; // must be multiple of 2 + static constexpr int kDropoutPipeCount = + 6; // must be multiple of 2 and less than or equal + // kConvertPipeCount + static constexpr int kAddPipeCount = + 4; // must be multiple of 2 and less than kDropoutPipeCount + + // To better overlap EX2 MUFU and emulation, we need to have last + // kE2eRes of every kE2eFreq elements in a row to use emulation while + // the rest go to MUFU. Also, we need to avoid using emulation after + // kE2eLimit elements as it would delay the execution of row sum of + // another softmax warpgroup because of resource contention + static constexpr int kE2eFreq = 16; + static constexpr int kE2eRes = 8; + static constexpr int kE2eLimit = BMM1_TILE_N / 16 * 9; + +#pragma unroll + for (int i = 0; i < kFmaPipeCount; i += 2) { + float2 in = make_float2(reinterpret_cast(reg_8_0[i]), + reinterpret_cast(reg_8_0[i + 1])); + float2 out = ffma2(bmm_scale_x_ln2_x2, in, minus_scaled_max_x2); + + reinterpret_cast(reg_8_0[i + 0]) = out.x; + reinterpret_cast(reg_8_0[i + 1]) = out.y; + } + + // Optimized version of alpha = __expf(current_max - total_max); + float exp_input = (total_max == NEG_INFINITY) + ? NEG_INFINITY + : (current_max - total_max); + float alpha = exp2f(exp_input); + + // Send stats to correction warp + reinterpret_cast(fp32_stats[0]) = alpha; + + // STTM + sttm_32dp32bit_x1(tmem_Stats, &fp32_stats[0]); + fence_view_async_tmem_store(); + arrive_barrier(cast_smem_ptr_to_uint(stat_full_mbar)); + +#pragma unroll + for (int i = 0; i < BMM1_TILE_N; i += 2) { + if (i - kConvertPipeCount == 32) { + sttm_32dp32bit_x16(tmem_fp16_S, ®_12_0[0]); + } + if (i - kConvertPipeCount == 64) { + sttm_32dp32bit_x16(tmem_fp16_S + 16, ®_12_0[16]); + fence_view_async_tmem_store(); + arrive_barrier(cast_smem_ptr_to_uint(bmm2_mbar + 0)); + } + + if (i >= kConvertPipeCount) { + fp32x2_to_bf16x2( + ®_12_0[((i - kConvertPipeCount) / 2) % 32], + reinterpret_cast(®_8_0[i - kConvertPipeCount])); + } + + reinterpret_cast(reg_8_0[i + 0]) = + exp2f(reinterpret_cast(reg_8_0[i + 0])); + + + if (i + kFmaPipeCount < BMM1_TILE_N) { + float2 in = make_float2( + reinterpret_cast(reg_8_0[i + kFmaPipeCount + 0]), + reinterpret_cast(reg_8_0[i + kFmaPipeCount + 1])); + float2 out = ffma2(bmm_scale_x_ln2_x2, in, minus_scaled_max_x2); + + reinterpret_cast(reg_8_0[i + kFmaPipeCount + 0]) = out.x; + reinterpret_cast(reg_8_0[i + kFmaPipeCount + 1]) = out.y; + } + + reinterpret_cast(reg_8_0[i + 1]) = + exp2f(reinterpret_cast(reg_8_0[i + 1])); + + if (i == kE2eLimit) { + if (softmax_gid == 1) { + named_barrier_arrive(SOFTMAX_BARRIER, 256); + named_barrier_wait(SOFTMAX_BARRIER + 1, 256); + } + } + } + +#pragma unroll + for (int i = BMM1_TILE_N - kConvertPipeCount; i < BMM1_TILE_N; + i += 2) { + fp32x2_to_bf16x2(®_12_0[(i / 2) % 32], + reinterpret_cast(®_8_0[i])); + } + sttm_32dp32bit_x32(tmem_fp16_S + BMM1_TILE_N / 64 * 16, + ®_12_0[((BMM1_TILE_N / 64) % 2) * 16]); + fence_view_async_tmem_store(); + arrive_barrier( + cast_smem_ptr_to_uint(bmm2_mbar + (BMM1_TILE_N / 64 - 1))); + + stat_mbar_state ^= 1; + if (!smem_bar_peek(cast_smem_ptr_to_uint(&stat_empty_mbar), + stat_mbar_state)) { + wait_barrier(cast_smem_ptr_to_uint(&stat_empty_mbar), + stat_mbar_state); + } + + float2 alpha2 = make_float2(alpha, alpha); + float local_row_sum_0 = 0.0f; + total_sum = fmul2(alpha2, total_sum); + +#pragma unroll + for (int i = 0; i < BMM1_TILE_N; i += 2) { + float2 in = make_float2(reinterpret_cast(reg_8_0[i + 0]), + reinterpret_cast(reg_8_0[i + 1])); + total_sum = fadd2(total_sum, in); + } + if (softmax_gid == 0) { + named_barrier_arrive(SOFTMAX_BARRIER + 1, 256); + named_barrier_wait(SOFTMAX_BARRIER, 256); + } + named_barrier_wait(SOFTMAX_BARRIER + 2 + softmax_gid, 128); + } + bmm_mbar_state ^= 1; + } // End of mainloop +#pragma unroll 1 + for (int kv_loop = FORT_MAX( + kv_loop_right_bound - + FORT_DIV_UP(BMM1_TILE_M * CGA_M_1 * Tiles_Q, BMM1_TILE_N), + kv_loop_left_bound); + kv_loop < kv_loop_right_bound; kv_loop += 1) { + int p_col_1 = kv_loop * TILE_N_1; + { + + wait_barrier(cast_smem_ptr_to_uint(&bmm1_mbar), bmm_mbar_state); + + r32 stat_reg[BMM1_TILE_N / 64]; +#pragma unroll + for (int ldtm_step = 0; ldtm_step < BMM1_TILE_N / 32; ldtm_step++) { + ldtm_32dp32bit_x32(®_8_0[32 * ldtm_step], + tmem_fp32_S + ldtm_step * 32); + } + constexpr bool use_ldtm_stat_max = false; + +)KERNEL" + R"KERNEL( + // global_load_op 10 code + + // mha_mask_op 11 code + + { +#pragma unroll + for (int i = 0; i < BMM1_TILE_N; i++) { + int row = p_row_1; + int col = p_col_1 + i; + bool mask = compute_diagonal_band_mask_11( + row, col, actual_seqlen_kv_1, actual_seqlen_q_1, + shift_right_bound, left_bound); + if (!mask) { + reinterpret_cast(reg_8_0[i]) = tensor_13; + } + } + } + + // softmax_op 12 code + float current_max = + row_max_reduction_128_elems(reg_8_0) * total_max_scale_12; + + // Update total max + float total_max_tmp = fmaxf(current_max, total_max); + current_max = total_max; + total_max = total_max_tmp; + + float2 bmm_scale_x_ln2_x2 = + make_float2(total_max_scale_12, total_max_scale_12); + float total_max_scaled = + (total_max == NEG_INFINITY) ? 0.0f : total_max; + float2 minus_scaled_max_x2 = + make_float2(-total_max_scaled, -total_max_scaled); + + static constexpr int kMulPipeCount = 14; // must be multiple of 2 + static constexpr int kSubtractPipeCount = 12; // must be multiple of 2 + static constexpr int kFmaPipeCount = 12; // must be multiple of 2 + static constexpr int kConvertPipeCount = 8; // must be multiple of 2 + static constexpr int kReleasePipeCount = 4; // must be multiple of 2 + static constexpr int kDropoutPipeCount = + 6; // must be multiple of 2 and less than or equal + // kConvertPipeCount + static constexpr int kAddPipeCount = + 4; // must be multiple of 2 and less than kDropoutPipeCount + + // To better overlap EX2 MUFU and emulation, we need to have last + // kE2eRes of every kE2eFreq elements in a row to use emulation while + // the rest go to MUFU. Also, we need to avoid using emulation after + // kE2eLimit elements as it would delay the execution of row sum of + // another softmax warpgroup because of resource contention + static constexpr int kE2eFreq = 16; + static constexpr int kE2eRes = 8; + static constexpr int kE2eLimit = BMM1_TILE_N / 16 * 9; + +#pragma unroll + for (int i = 0; i < kFmaPipeCount; i += 2) { + float2 in = make_float2(reinterpret_cast(reg_8_0[i]), + reinterpret_cast(reg_8_0[i + 1])); + float2 out = ffma2(bmm_scale_x_ln2_x2, in, minus_scaled_max_x2); + + reinterpret_cast(reg_8_0[i + 0]) = out.x; + reinterpret_cast(reg_8_0[i + 1]) = out.y; + } + + // Optimized version of alpha = __expf(current_max - total_max); + float exp_input = (total_max == NEG_INFINITY) + ? NEG_INFINITY + : (current_max - total_max); + float alpha = exp2f(exp_input); + + // Send stats to correction warp + reinterpret_cast(fp32_stats[0]) = alpha; + + // STTM + sttm_32dp32bit_x1(tmem_Stats, &fp32_stats[0]); + fence_view_async_tmem_store(); + arrive_barrier(cast_smem_ptr_to_uint(stat_full_mbar)); + +#pragma unroll + for (int i = 0; i < BMM1_TILE_N; i += 2) { + if (i - kConvertPipeCount == 32) { + sttm_32dp32bit_x16(tmem_fp16_S, ®_12_0[0]); + } + if (i - kConvertPipeCount == 64) { + sttm_32dp32bit_x16(tmem_fp16_S + 16, ®_12_0[16]); + fence_view_async_tmem_store(); + arrive_barrier(cast_smem_ptr_to_uint(bmm2_mbar + 0)); + } + + if (i >= kConvertPipeCount) { + fp32x2_to_bf16x2( + ®_12_0[((i - kConvertPipeCount) / 2) % 32], + reinterpret_cast(®_8_0[i - kConvertPipeCount])); + } + + reinterpret_cast(reg_8_0[i + 0]) = + exp2f(reinterpret_cast(reg_8_0[i + 0])); + + + if (i + kFmaPipeCount < BMM1_TILE_N) { + float2 in = make_float2( + reinterpret_cast(reg_8_0[i + kFmaPipeCount + 0]), + reinterpret_cast(reg_8_0[i + kFmaPipeCount + 1])); + float2 out = ffma2(bmm_scale_x_ln2_x2, in, minus_scaled_max_x2); + + reinterpret_cast(reg_8_0[i + kFmaPipeCount + 0]) = out.x; + reinterpret_cast(reg_8_0[i + kFmaPipeCount + 1]) = out.y; + } + + reinterpret_cast(reg_8_0[i + 1]) = + exp2f(reinterpret_cast(reg_8_0[i + 1])); + + if (i == kE2eLimit) { + if (softmax_gid == 1) { + named_barrier_arrive(SOFTMAX_BARRIER, 256); + named_barrier_wait(SOFTMAX_BARRIER + 1, 256); + } + } + } + +#pragma unroll + for (int i = BMM1_TILE_N - kConvertPipeCount; i < BMM1_TILE_N; + i += 2) { + fp32x2_to_bf16x2(®_12_0[(i / 2) % 32], + reinterpret_cast(®_8_0[i])); + } + sttm_32dp32bit_x32(tmem_fp16_S + BMM1_TILE_N / 64 * 16, + ®_12_0[((BMM1_TILE_N / 64) % 2) * 16]); + fence_view_async_tmem_store(); + arrive_barrier( + cast_smem_ptr_to_uint(bmm2_mbar + (BMM1_TILE_N / 64 - 1))); + + stat_mbar_state ^= 1; + if (!smem_bar_peek(cast_smem_ptr_to_uint(&stat_empty_mbar), + stat_mbar_state)) { + wait_barrier(cast_smem_ptr_to_uint(&stat_empty_mbar), + stat_mbar_state); + } + + float2 alpha2 = make_float2(alpha, alpha); + float local_row_sum_0 = 0.0f; + total_sum = fmul2(alpha2, total_sum); + +#pragma unroll + for (int i = 0; i < BMM1_TILE_N; i += 2) { + float2 in = make_float2(reinterpret_cast(reg_8_0[i + 0]), + reinterpret_cast(reg_8_0[i + 1])); + total_sum = fadd2(total_sum, in); + } + if (softmax_gid == 0) { + named_barrier_arrive(SOFTMAX_BARRIER + 1, 256); + named_barrier_wait(SOFTMAX_BARRIER, 256); + } + named_barrier_wait(SOFTMAX_BARRIER + 2 + softmax_gid, 128); + } + bmm_mbar_state ^= 1; + } // End of mainloop + + float final_total_sum = total_sum.x + total_sum.y; + + reinterpret_cast(fp32_stats[0]) = total_max; + reinterpret_cast(fp32_stats[1]) = final_total_sum; + // STTM + sttm_32dp32bit_x2(tmem_Stats, &fp32_stats[0]); + fence_view_async_tmem_store(); + arrive_barrier(cast_smem_ptr_to_uint(stat_full_mbar)); + + // This portion of the code may seem strange, please read the + // Function: work_tile_info_from_workid_response, load_query_response + if (elect_one) { + arrive_barrier(cast_smem_ptr_to_uint( + &(shared_storage.read_tile_id_done_mbar[sched_state.index()]))); + } + // Get next tile id + wait_barrier(cast_smem_ptr_to_uint( + &shared_storage.scheduler_mbar[sched_state.index()]), + sched_state.phase()); + uint32_t tile_id_smem_int_ptr = cast_smem_ptr_to_uint( + &shared_storage.tile_id[sched_state.index() * 8]); + lds_128(tile_id_and_coord, tile_id_smem_int_ptr); + ++sched_state; + + uint32_t tile_idx = tile_id_and_coord[0]; + uint32_t bidhb, l2_mod; + uint32_t block = 0, bidhb_residual = 0; + + fastDivMod(l2_major_divmod, tile_idx, bidhb, l2_mod); + if (bidhb < num_hb_quotient) { + { + fastDivMod(l2_minor_divmod, l2_mod, block, bidhb_residual); + } + } else { + { + fastDivMod(l2_minor_residual_divmod, l2_mod, block, bidhb_residual); + } + } + + int bidhb_actual = bidhb * l2_minor_divmod.val + bidhb_residual; + uint32_t batch_idx, head_idx; + fastDivMod(num_head_divmod, bidhb_actual, batch_idx, head_idx); + block = num_block - 1 - block; + + lean_tile_id = tile_id_and_coord[0]; + + blocked_row_coord = block + 0; + head_coord_from_grid_1 = (head_idx) + 0; + batch_coord_1 = batch_idx; + is_valid_tile = tile_id_and_coord[2] & 1; + + head_coord_1 = head_coord_from_grid_1; + head_coord_k_1 = head_coord_1 / attnDesc.q_heads_per_k; + head_coord_v_1 = head_coord_1 / attnDesc.q_heads_per_v; + + q_row_coord = (blocked_row_coord * Tiles_Q * BMM1_TILE_M); + int2 kv_loop_bounds = compute_kv_loop_bounds( + q_row_coord, BMM1_TILE_M * Tiles_Q * CGA_M_1, TILE_N_1, + actual_seqlen_kv_1, actual_seqlen_q_1, shift_right_bound, left_bound); + kv_loop_left_bound = kv_loop_bounds.x; + kv_loop_right_bound = kv_loop_bounds.y; + if (actual_seqlen_q_1 == 0 || q_row_coord >= actual_seqlen_q_1) { + kv_loop_right_bound = kv_loop_left_bound; + } + } // end of persistent loop + } + + // thread_specialize_op 22 code + else if (wid >= SoftmaxWarps && wid < SoftmaxWarps + CorrectionWarps) { + // Correction + reg_dealloc(); + uint32_t base_tmem_addr; + named_barrier_wait(TMEM_ALLOC_CORRECTION_BARRIER, + threads_per_warp * (CorrectionWarps + 1)); + lds_32(&base_tmem_addr, + cast_smem_ptr_to_uint(&shared_storage.tmem_base_ptrs)); + uint32_t tmem_fp32_O_0 = base_tmem_addr + 256; + uint32_t tmem_fp32_O_1 = base_tmem_addr + 384; + uint32_t tmem_Stats_0 = base_tmem_addr + 0; + uint32_t tmem_Stats_1 = base_tmem_addr + 0 + BMM1_TILE_N; + + static constexpr int local_correction_block_size = 16; + static constexpr int local_epilogue_block_size = + 128 / sizeof(ElementO); // hack as only K=128/64 supported. Both can do + // 128B TMASTG. + +)KERNEL" + R"KERNEL( + float amax_s_1 = 0.f; + float amax_o_1 = 0.f; + + // Registers + uint32_t fp32_stats[2]; + uint32_t fp32_O[local_correction_block_size]; + + // Memory barrier states + uint32_t epilogue_state = 0; + uint32_t bmm_mbar_state = 0; + uint32_t stat_mbar_state = 0; + +// Persistent loop over tiles +#pragma unroll 1 + while (is_valid_tile) { + if (elect_one) { + arrive_barrier(cast_smem_ptr_to_uint( + &(shared_storage.read_tile_id_done_mbar[sched_state.index()]))); + } + + if (kv_loop_left_bound < kv_loop_right_bound) { + // No correction for iter 0 + arrive_barrier( + cast_smem_ptr_to_uint(&(shared_storage.bmm2_tile0_ready_mbar[0]))); + arrive_barrier( + cast_smem_ptr_to_uint(&(shared_storage.bmm2_tile1_ready_mbar[0]))); + + { + wait_barrier( + cast_smem_ptr_to_uint(&shared_storage.stat_tile0_full_mbar[0]), + stat_mbar_state); + arrive_barrier(cast_smem_ptr_to_uint( + &(shared_storage.stat_tile0_empty_mbar[0]))); + } + { + wait_barrier( + cast_smem_ptr_to_uint(&shared_storage.stat_tile1_full_mbar[0]), + stat_mbar_state); + arrive_barrier(cast_smem_ptr_to_uint( + &(shared_storage.stat_tile1_empty_mbar[0]))); + } + stat_mbar_state ^= 1; + } else { + arrive_barrier( + cast_smem_ptr_to_uint(&(shared_storage.empty_mainloop_mbar[0]))); + } + +#pragma unroll 1 + for (int kv_loop = kv_loop_left_bound + 1; kv_loop < kv_loop_right_bound; + kv_loop += 1) { +#pragma unroll + for (int sub_tile_id = 0; sub_tile_id < 2; ++sub_tile_id) { + uint32_t tmem_Stats = sub_tile_id == 0 ? tmem_Stats_0 : tmem_Stats_1; + uint32_t tmem_fp32_O = + sub_tile_id == 0 ? tmem_fp32_O_0 : tmem_fp32_O_1; + + uint64_t &stat_full_mbar = + sub_tile_id == 0 ? shared_storage.stat_tile0_full_mbar[0] + : shared_storage.stat_tile1_full_mbar[0]; + uint64_t &bmm_done_mbar = + sub_tile_id == 0 ? shared_storage.bmm2_tile0_done_mbar[0] + : shared_storage.bmm2_tile1_done_mbar[0]; + uint64_t *stat_empty_mbar = + sub_tile_id == 0 ? &(shared_storage.stat_tile0_empty_mbar[0]) + : &(shared_storage.stat_tile1_empty_mbar[0]); + uint64_t *bmm_ready_mbar = + sub_tile_id == 0 ? &(shared_storage.bmm2_tile0_ready_mbar[0]) + : &(shared_storage.bmm2_tile1_ready_mbar[0]); + + wait_barrier(cast_smem_ptr_to_uint(&stat_full_mbar), stat_mbar_state); + ldtm_32dp32bit_x1(&fp32_stats[0], tmem_Stats); + + float alpha = reinterpret_cast(fp32_stats[0]); + fence_view_async_tmem_load(); + arrive_barrier(cast_smem_ptr_to_uint(stat_empty_mbar)); + + bool alpha_is_one = (alpha == 1.0f); + bool all_alpha_one = __all_sync(0xFFFFFFFF, alpha_is_one); + + float2 alpha2 = make_float2(alpha, alpha); + + wait_barrier(cast_smem_ptr_to_uint(&bmm_done_mbar), bmm_mbar_state); + + if (!all_alpha_one) { +#pragma unroll + for (int block = 0; block < TILE_O_1 / local_correction_block_size; + block++) { +// LDTM +#pragma unroll + for (int ldtm_step = 0; + ldtm_step < local_correction_block_size / 8; ldtm_step++) { + ldtm_32dp32bit_x8(&fp32_O[8 * ldtm_step], + tmem_fp32_O + + block * local_correction_block_size + + ldtm_step * 8); + } + +#pragma unroll + for (int i = 0; i < local_correction_block_size; i += 2) { + float2 in = + make_float2(reinterpret_cast(fp32_O[i + 0]), + reinterpret_cast(fp32_O[i + 1])); + float2 out = fmul2(alpha2, in); + reinterpret_cast(fp32_O[i + 0]) = out.x; + reinterpret_cast(fp32_O[i + 1]) = out.y; + } + +#pragma unroll + for (int sttm_step = 0; + sttm_step < local_correction_block_size / 8; sttm_step++) { + sttm_32dp32bit_x8(tmem_fp32_O + + block * local_correction_block_size + + sttm_step * 8, + &fp32_O[8 * sttm_step]); + } + } + } + fence_view_async_tmem_store(); + arrive_barrier(cast_smem_ptr_to_uint(bmm_ready_mbar)); + } + stat_mbar_state ^= 1; + bmm_mbar_state ^= 1; + } // End of mainloop + + // Epilogue + { + epilogue_state ^= 1; + + static constexpr int stg_step = 16 / sizeof(ElementO); + const int local_row = (tid % 8) / 1; + + // global_store_op 13 decls + int oob_M_13 = desc_7.dims[2]; + int row_13 = 0; + char *ptr_13 = + reinterpret_cast(tensor_7) + + batch_coord_1 * desc_7.strides[0] * BYTES_PER_ELEMENT_13 + + head_coord_1 * desc_7.strides[1] * BYTES_PER_ELEMENT_13 + + row_13 * desc_7.strides[2] * BYTES_PER_ELEMENT_13; + + // global_store_op 14 decls + int oob_M_14 = desc_6.dims[2]; + int row_14 = 0; + char *ptr_14 = + reinterpret_cast(tensor_6) + + batch_coord_1 * desc_6.strides[0] * BYTES_PER_ELEMENT_14 + + head_coord_1 * desc_6.strides[1] * BYTES_PER_ELEMENT_14 + + row_14 * desc_6.strides[2] * BYTES_PER_ELEMENT_14; + +#pragma unroll + for (int sub_tile_id = 0; sub_tile_id < 2; ++sub_tile_id) { + const int row_coord = q_row_coord + sub_tile_id * TILE_M_1 + tid; + ElementO *smem_O = (sub_tile_id == 0) ? shared_storage.smem_O + : shared_storage.smem_V; + + uint32_t fp32_reg_O[local_epilogue_block_size]; + uint32_t reg_O[local_epilogue_block_size / (4 / sizeof(ElementO))]; + + uint32_t tmem_Stats = sub_tile_id == 0 ? tmem_Stats_0 : tmem_Stats_1; + uint32_t tmem_fp32_O = + sub_tile_id == 0 ? tmem_fp32_O_0 : tmem_fp32_O_1; + + uint64_t &stat_full_mbar = + sub_tile_id == 0 ? shared_storage.stat_tile0_full_mbar[0] + : shared_storage.stat_tile1_full_mbar[0]; + uint64_t &bmm_done_mbar = + sub_tile_id == 0 ? shared_storage.bmm2_tile0_done_mbar[0] + : shared_storage.bmm2_tile1_done_mbar[0]; + uint64_t &stat_empty_mbar = + sub_tile_id == 0 ? shared_storage.stat_tile0_empty_mbar[0] + : shared_storage.stat_tile1_empty_mbar[0]; + uint64_t &final_stat_empty_mbar = + sub_tile_id == 0 ? shared_storage.final_stat_tile0_empty_mbar[0] + : shared_storage.final_stat_tile1_empty_mbar[0]; + + wait_barrier(cast_smem_ptr_to_uint(&stat_full_mbar), stat_mbar_state); + ldtm_32dp32bit_x2(&fp32_stats[0], tmem_Stats); + + float total_max = reinterpret_cast(fp32_stats[0]) * ln2; + float total_sum = reinterpret_cast(fp32_stats[1]); + fence_view_async_tmem_load(); + arrive_barrier(cast_smem_ptr_to_uint(&stat_empty_mbar)); + arrive_barrier(cast_smem_ptr_to_uint(&final_stat_empty_mbar)); + + float softmax_stats = + total_sum == 0.f ? NEG_INFINITY : total_max + __logf(total_sum); + + // global_store_op 13 code + if (row_coord < actual_seqlen_q_1) { + stg_32(ptr_13 + + row_coord * desc_7.strides[2] * BYTES_PER_ELEMENT_13, + &(reinterpret_cast(total_max))); + } + + // global_store_op 14 code + if (row_coord < actual_seqlen_q_1) { + stg_32(ptr_14 + + row_coord * desc_6.strides[2] * BYTES_PER_ELEMENT_14, + &(reinterpret_cast(total_sum))); + } + + float threshold_beta = (total_sum == 0.f) ? 0.f : 1.f / total_sum; + float beta = threshold_beta; + + float2 beta2 = make_float2(threshold_beta, threshold_beta); + + wait_barrier(cast_smem_ptr_to_uint(&bmm_done_mbar), bmm_mbar_state); + + if (sub_tile_id == 0) { + wait_barrier( + cast_smem_ptr_to_uint(&shared_storage.tma_o_0_empty_mbar[0]), + epilogue_state); + } +#pragma unroll + for (int block = 0; block < TILE_O_1 / local_epilogue_block_size; + block++) { + { +#pragma unroll + for (int ldtm_step = 0; + ldtm_step < local_epilogue_block_size / 32; ldtm_step++) { + ldtm_32dp32bit_x32(&fp32_reg_O[32 * ldtm_step], + tmem_fp32_O + + block * local_epilogue_block_size + + ldtm_step * 32); + } + } + +#pragma unroll + for (int i = 0; i < local_epilogue_block_size; i += 2) { + float2 in = + make_float2(reinterpret_cast(fp32_reg_O[i + 0]), + reinterpret_cast(fp32_reg_O[i + 1])); + float2 out = fmul2(beta2, in); + reinterpret_cast(fp32_reg_O[i + 0]) = out.x; + reinterpret_cast(fp32_reg_O[i + 1]) = out.y; + } + +// general_data_type_cast code +#pragma unroll + for (int i = 0; i < local_epilogue_block_size; i += 2) { + fp32x2_to_bf16x2(®_O[i / 2], &fp32_reg_O[i]); + } + +#pragma unroll + for (int i = 0; i < local_epilogue_block_size / stg_step; i++) { + char *smem_loc = get_smem_loc_epilogue_swizzle_128b( + reinterpret_cast(smem_O), block, tid, local_row, i, + sizeof(ElementO), local_epilogue_block_size, TILE_M_1); +)KERNEL" + R"KERNEL( + sts_128(cast_smem_ptr_to_uint(smem_loc), + reinterpret_cast(®_O[i * 4])); + } + + uint64_t *tma_o_full_mbar = + sub_tile_id == 0 ? &(shared_storage.tma_o_0_full_mbar[block]) + : &(shared_storage.tma_o_1_full_mbar[block]); + fence_view_async_shared(); + arrive_barrier(cast_smem_ptr_to_uint(tma_o_full_mbar)); + } + } + stat_mbar_state ^= 1; + bmm_mbar_state ^= 1; + } // Epilogue end + + // Get next tile id + wait_barrier(cast_smem_ptr_to_uint( + &shared_storage.scheduler_mbar[sched_state.index()]), + sched_state.phase()); + uint32_t tile_id_smem_int_ptr = cast_smem_ptr_to_uint( + &shared_storage.tile_id[sched_state.index() * 8]); + lds_128(tile_id_and_coord, tile_id_smem_int_ptr); + ++sched_state; + + uint32_t tile_idx = tile_id_and_coord[0]; + uint32_t bidhb, l2_mod; + uint32_t block = 0, bidhb_residual = 0; + + fastDivMod(l2_major_divmod, tile_idx, bidhb, l2_mod); + if (bidhb < num_hb_quotient) { + { + fastDivMod(l2_minor_divmod, l2_mod, block, bidhb_residual); + } + } else { + { + fastDivMod(l2_minor_residual_divmod, l2_mod, block, bidhb_residual); + } + } + + int bidhb_actual = bidhb * l2_minor_divmod.val + bidhb_residual; + uint32_t batch_idx, head_idx; + fastDivMod(num_head_divmod, bidhb_actual, batch_idx, head_idx); + block = num_block - 1 - block; + + lean_tile_id = tile_id_and_coord[0]; + + blocked_row_coord = block + 0; + head_coord_from_grid_1 = (head_idx) + 0; + batch_coord_1 = batch_idx; + is_valid_tile = tile_id_and_coord[2] & 1; + + head_coord_1 = head_coord_from_grid_1; + head_coord_k_1 = head_coord_1 / attnDesc.q_heads_per_k; + head_coord_v_1 = head_coord_1 / attnDesc.q_heads_per_v; + + q_row_coord = (blocked_row_coord * Tiles_Q * BMM1_TILE_M); + int2 kv_loop_bounds = compute_kv_loop_bounds( + q_row_coord, BMM1_TILE_M * Tiles_Q * CGA_M_1, TILE_N_1, + actual_seqlen_kv_1, actual_seqlen_q_1, shift_right_bound, left_bound); + kv_loop_left_bound = kv_loop_bounds.x; + kv_loop_right_bound = kv_loop_bounds.y; + if (actual_seqlen_q_1 == 0 || q_row_coord >= actual_seqlen_q_1) { + kv_loop_right_bound = kv_loop_left_bound; + } + } // end of persistent loop + arrive_barrier( + cast_smem_ptr_to_uint(&shared_storage.tmem_dealloc_smem_bar[0])); + } + + // thread_specialize_op 23 code + else if (wid == SoftmaxWarps + CorrectionWarps) { + // MMA + reg_dealloc(); + tmem_allocate_1sm(num_columns_per_tmem, + cast_smem_ptr_to_uint(&shared_storage.tmem_base_ptrs)); + tmem_release_allocation_lock_1sm(); + __syncwarp(); + named_barrier_arrive(TMEM_ALLOC_SOFTMAX_BARRIER, + threads_per_warp * (SoftmaxWarps + 1)); + named_barrier_arrive(TMEM_ALLOC_CORRECTION_BARRIER, + threads_per_warp * (CorrectionWarps + 1)); + uint32_t base_tmem_addr; + lds_32(&base_tmem_addr, + cast_smem_ptr_to_uint(&shared_storage.tmem_base_ptrs)); + + uint32_t tmem_S_acc_0 = base_tmem_addr + 0; + uint32_t tmem_S_acc_1 = base_tmem_addr + BMM1_TILE_N; + + uint32_t tmem_S_bmm2_0 = base_tmem_addr + BMM1_TILE_N / 2; + uint32_t tmem_S_bmm2_1 = base_tmem_addr + 3 * BMM1_TILE_N / 2; + + uint32_t tmem_O_0 = base_tmem_addr + 256; + uint32_t tmem_O_1 = base_tmem_addr + 384; + + // Memory barrier states + uint32_t bmm_mbar_state = 0; + uint32_t final_stat_mbar_state = 1; + uint32_t empty_mainloop_mbar_state = 0; + + PipelineState kv_mbar_state(0, 0); // index, state + PipelineState q_mbar_state(0, 0); // index, state + + // mma_op 8 decls + constexpr uint64_t utcmma_instruction_desc_8 = + build_utcmma_instruction_desc<0, // max shift for WS + TILE_M_1 * CTA_MMA_1, TILE_N_1, + 0, // Transpose for B + 0, // Transpose for A + 0, // Negate for B + 0, // Negate for A + 1, // Data type B + 1, // Data type A + 0, // sparse meta data format + 1, // Acc type fp32 + 0, // saturate disable + 0, // sparse disable + 0 // sparse meta data id2 ??? + >(); + // Swizzle mode, base smem offset, desc type, bytes per leading dim, tile k + Smem_utcmma_descriptor utcmma_smem_desc_Q_0(2, 0, 1, 16, + ROWS_PER_CORE_MATRIX_A_8 * 128); + Smem_utcmma_descriptor utcmma_smem_desc_K(2, 0, 1, 16, + ROWS_PER_CORE_MATRIX_B_8 * 128); + + utcmma_smem_desc_Q_0.set_smem( + cast_smem_ptr_to_uint(shared_storage.smem_Q)); + utcmma_smem_desc_K.set_smem( + cast_smem_ptr_to_uint(shared_storage.smem_K)); + + // mma_op 17 decls + constexpr uint64_t utcmma_instruction_desc_17 = + build_utcmma_instruction_desc<0, // max shift for WS + TILE_M_1 * CTA_MMA_1, TILE_O_1, + 1, // Transpose for B + 0, // Transpose for A + 0, // Negate for B + 0, // Negate for A + 1, // Data type B + 1, // Data type A + 0, // sparse meta data format + 1, // Acc type fp32 + 0, // saturate disable + 0, // sparse disable + 0 // sparse meta data id2 ??? + >(); + // Swizzle mode, base smem offset, desc type, bytes per leading dim, tile k + Smem_utcmma_descriptor utcmma_smem_desc_V( + 2, 0, 1, TILE_O_1 / ROWS_PER_CORE_MATRIX_B_17 <= 8 ? 0 : TILE_N_1 * 128, + COLS_PER_CORE_MATRIX_B_17 * 128); + + utcmma_smem_desc_V.set_smem( + cast_smem_ptr_to_uint(shared_storage.smem_V)); + +// Persistent loop over tiles +#pragma unroll 1 + while (is_valid_tile) { + if (elect_one) { + arrive_barrier(cast_smem_ptr_to_uint( + &(shared_storage.read_tile_id_done_mbar[sched_state.index()]))); + } + //---------------------------------------------------------// + //------------------- MMA Pipeline Begin ------------------// + //---------------------------------------------------------// + + if (kv_loop_left_bound < kv_loop_right_bound) { + wait_barrier(cast_smem_ptr_to_uint( + &shared_storage.final_stat_tile0_empty_mbar[0]), + final_stat_mbar_state); + + const int kv_stage = kv_mbar_state.index(); + const int kv_state = kv_mbar_state.phase(); + wait_barrier( + cast_smem_ptr_to_uint(&shared_storage.tma_k_full_mbar[kv_stage]), + kv_state); + + const int q_stage = q_mbar_state.index(); + const int q_phase = q_mbar_state.phase(); + wait_barrier( + cast_smem_ptr_to_uint(&shared_storage.tma_q_full_mbar[q_stage]), + q_phase); + ++q_mbar_state; + + // Q * K matmul for sub-tile 0 + { + Smem_utcmma_descriptor &utcmma_smem_desc_Q = utcmma_smem_desc_Q_0; + uint32_t tmem_acc = tmem_S_acc_0; + + // mma_op 8 code +#pragma unroll + for (int k = 0; k < BMM1_XMMAS_K; ++k) { + if (elect_one) + utcmma_asmem_bsmem_h_cta1(utcmma_smem_desc_Q.desc, + utcmma_smem_desc_K.desc, tmem_acc, + utcmma_instruction_desc_8, (k > 0)); + if ((k % 4) == ((BMM1_XMMAS_K - 1) % 4)) { + utcmma_smem_desc_Q.add_smem_offset<-BYTES_PER_MMA_K *( + (BMM1_XMMAS_K - 1) % 4)>(); + utcmma_smem_desc_Q.add_smem_offset<128 * BMM1_TILE_M>(); + } else { + utcmma_smem_desc_Q.add_smem_offset(); + } + if ((k % 4) == ((BMM1_XMMAS_K - 1) % 4)) { + utcmma_smem_desc_K.add_smem_offset<-BYTES_PER_MMA_K *( + (BMM1_XMMAS_K - 1) % 4)>(); + utcmma_smem_desc_K + .add_smem_offset<128 * BMM1_TILE_N / CTA_MMA_1>(); + } else { + utcmma_smem_desc_K.add_smem_offset(); + } + } + utcmma_smem_desc_Q.add_smem_offset<-128 * BMM1_TILE_M * + FORT_MAX(BMM1_XMMAS_K / 4, 1)>(); + utcmma_smem_desc_K.add_smem_offset<-128 * BMM1_TILE_N / CTA_MMA_1 * + FORT_MAX(BMM1_XMMAS_K / 4, 1)>(); + // change the smem buffer for Q + utcmma_smem_desc_Q + .increment_smem_buffer(); + } + + if (elect_one) { + umma_arrive( + cast_smem_ptr_to_uint(&shared_storage.bmm1_tile0_done_mbar[0])); + } // elect_one + } // if (kv_loop_left_bound < kv_loop_right_bound) + if (kv_loop_left_bound < kv_loop_right_bound) { + wait_barrier(cast_smem_ptr_to_uint( + &shared_storage.final_stat_tile1_empty_mbar[0]), + final_stat_mbar_state); + + const int kv_stage = kv_mbar_state.index(); + const int kv_state = kv_mbar_state.phase(); + const int q_stage = q_mbar_state.index(); + const int q_phase = q_mbar_state.phase(); + wait_barrier( + cast_smem_ptr_to_uint(&shared_storage.tma_q_full_mbar[q_stage]), + q_phase); + ++q_mbar_state; + + // Q * K matmul for sub-tile 1 + { + Smem_utcmma_descriptor &utcmma_smem_desc_Q = utcmma_smem_desc_Q_0; + uint32_t tmem_acc = tmem_S_acc_1; + + // mma_op 8 code +#pragma unroll + for (int k = 0; k < BMM1_XMMAS_K; ++k) { + if (elect_one) + utcmma_asmem_bsmem_h_cta1(utcmma_smem_desc_Q.desc, +)KERNEL" + R"KERNEL( + utcmma_smem_desc_K.desc, tmem_acc, + utcmma_instruction_desc_8, (k > 0)); + if ((k % 4) == ((BMM1_XMMAS_K - 1) % 4)) { + utcmma_smem_desc_Q.add_smem_offset<-BYTES_PER_MMA_K *( + (BMM1_XMMAS_K - 1) % 4)>(); + utcmma_smem_desc_Q.add_smem_offset<128 * BMM1_TILE_M>(); + } else { + utcmma_smem_desc_Q.add_smem_offset(); + } + if ((k % 4) == ((BMM1_XMMAS_K - 1) % 4)) { + utcmma_smem_desc_K.add_smem_offset<-BYTES_PER_MMA_K *( + (BMM1_XMMAS_K - 1) % 4)>(); + utcmma_smem_desc_K + .add_smem_offset<128 * BMM1_TILE_N / CTA_MMA_1>(); + } else { + utcmma_smem_desc_K.add_smem_offset(); + } + } + utcmma_smem_desc_Q.add_smem_offset<-128 * BMM1_TILE_M * + FORT_MAX(BMM1_XMMAS_K / 4, 1)>(); + utcmma_smem_desc_K.add_smem_offset<-128 * BMM1_TILE_N / CTA_MMA_1 * + FORT_MAX(BMM1_XMMAS_K / 4, 1)>(); + // change the smem buffer for K + utcmma_smem_desc_K + .increment_smem_buffer(); + // change the smem buffer for Q + utcmma_smem_desc_Q + .increment_smem_buffer(); + const int kv_stage = kv_mbar_state.index(); + if (elect_one) + umma_arrive(cast_smem_ptr_to_uint( + &shared_storage.tma_k_empty_mbar[kv_stage])); + } + + if (elect_one) { + umma_arrive( + cast_smem_ptr_to_uint(&shared_storage.bmm1_tile1_done_mbar[0])); + } // elect_one + } + +// Mainloop +#pragma unroll 1 + for (int kv_loop = kv_loop_left_bound + 1; kv_loop < kv_loop_right_bound; + kv_loop += 1) { + const int kv_stage = kv_mbar_state.index(); + const uint32_t kv_state = kv_mbar_state.phase(); + + ++kv_mbar_state; + + wait_barrier( + cast_smem_ptr_to_uint(&shared_storage.tma_v_full_mbar[kv_stage]), + kv_state); + + // S * V matmul for sub-tile 0 + { + uint32_t bmm2_ready_mbar = + cast_smem_ptr_to_uint(&shared_storage.bmm2_tile0_ready_mbar[0]); + uint32_t tmem_O = tmem_O_0; + uint32_t tmem_S = tmem_S_bmm2_0; + uint32_t scaleC = kv_loop > kv_loop_left_bound + 1; + + // mma_op 17 code +#pragma unroll + for (int k = 0; k < BMM2_XMMAS_K; ++k) { + if (k % (BMM2_XMMAS_K / (BMM1_TILE_N / 64)) == 0) { + wait_barrier(bmm2_ready_mbar + + (k / (BMM2_XMMAS_K / (BMM1_TILE_N / 64))) * 8, + bmm_mbar_state); + } + if (elect_one) + utcmma_atmem_bsmem_h_cta1( + tmem_S + k * (UTCMMA_TILE_K / (4 / BYTES_PER_ELEMENT)), + utcmma_smem_desc_V.desc, tmem_O, utcmma_instruction_desc_17, + scaleC || k > 0); + if (k == BMM2_XMMAS_K - 1) { + utcmma_smem_desc_V + .add_smem_offset<-UTCMMA_TILE_K * 128 * (BMM2_XMMAS_K - 1)>(); + } else { + utcmma_smem_desc_V.add_smem_offset(); + } + } + if (elect_one) + umma_arrive( + cast_smem_ptr_to_uint(&shared_storage.bmm2_tile0_done_mbar[0])); + } + + wait_barrier( + cast_smem_ptr_to_uint( + &shared_storage.tma_k_full_mbar[kv_mbar_state.index()]), + kv_mbar_state.phase()); + + // Q * K matmul for sub-tile 0 + { + Smem_utcmma_descriptor &utcmma_smem_desc_Q = utcmma_smem_desc_Q_0; + uint32_t tmem_acc = tmem_S_acc_0; + + // mma_op 8 code +#pragma unroll + for (int k = 0; k < BMM1_XMMAS_K; ++k) { + if (elect_one) + utcmma_asmem_bsmem_h_cta1(utcmma_smem_desc_Q.desc, + utcmma_smem_desc_K.desc, tmem_acc, + utcmma_instruction_desc_8, (k > 0)); + if ((k % 4) == ((BMM1_XMMAS_K - 1) % 4)) { + utcmma_smem_desc_Q.add_smem_offset<-BYTES_PER_MMA_K *( + (BMM1_XMMAS_K - 1) % 4)>(); + utcmma_smem_desc_Q.add_smem_offset<128 * BMM1_TILE_M>(); + } else { + utcmma_smem_desc_Q.add_smem_offset(); + } + if ((k % 4) == ((BMM1_XMMAS_K - 1) % 4)) { + utcmma_smem_desc_K.add_smem_offset<-BYTES_PER_MMA_K *( + (BMM1_XMMAS_K - 1) % 4)>(); + utcmma_smem_desc_K + .add_smem_offset<128 * BMM1_TILE_N / CTA_MMA_1>(); + } else { + utcmma_smem_desc_K.add_smem_offset(); + } + } + utcmma_smem_desc_Q.add_smem_offset<-128 * BMM1_TILE_M * + FORT_MAX(BMM1_XMMAS_K / 4, 1)>(); + utcmma_smem_desc_K.add_smem_offset<-128 * BMM1_TILE_N / CTA_MMA_1 * + FORT_MAX(BMM1_XMMAS_K / 4, 1)>(); + // change the smem buffer for Q + utcmma_smem_desc_Q + .increment_smem_buffer(); + } + + if (elect_one) { + umma_arrive( + cast_smem_ptr_to_uint(&shared_storage.bmm1_tile0_done_mbar[0])); + } // elect_one + + // S * V matmul for sub-tile 1 + { + uint32_t bmm2_ready_mbar = + cast_smem_ptr_to_uint(&shared_storage.bmm2_tile1_ready_mbar[0]); + uint32_t tmem_O = tmem_O_1; + uint32_t tmem_S = tmem_S_bmm2_1; + uint32_t scaleC = kv_loop > kv_loop_left_bound + 1; + + // mma_op 17 code +#pragma unroll + for (int k = 0; k < BMM2_XMMAS_K; ++k) { + if (k % (BMM2_XMMAS_K / (BMM1_TILE_N / 64)) == 0) { + wait_barrier(bmm2_ready_mbar + + (k / (BMM2_XMMAS_K / (BMM1_TILE_N / 64))) * 8, + bmm_mbar_state); + } + if (elect_one) + utcmma_atmem_bsmem_h_cta1( + tmem_S + k * (UTCMMA_TILE_K / (4 / BYTES_PER_ELEMENT)), + utcmma_smem_desc_V.desc, tmem_O, utcmma_instruction_desc_17, + scaleC || k > 0); + if (k == BMM2_XMMAS_K - 1) { + utcmma_smem_desc_V + .add_smem_offset<-UTCMMA_TILE_K * 128 * (BMM2_XMMAS_K - 1)>(); + } else { + utcmma_smem_desc_V.add_smem_offset(); + } + } + if (elect_one) + umma_arrive( + cast_smem_ptr_to_uint(&shared_storage.bmm2_tile1_done_mbar[0])); + if (elect_one) + umma_arrive(cast_smem_ptr_to_uint( + &shared_storage.tma_v_empty_mbar[kv_stage])); + // Change the buffer for V + utcmma_smem_desc_V + .increment_smem_buffer(); + } + bmm_mbar_state ^= 1; + + // Q * K matmul for sub-tile 1 + { + Smem_utcmma_descriptor &utcmma_smem_desc_Q = utcmma_smem_desc_Q_0; + uint32_t tmem_acc = tmem_S_acc_1; + + // mma_op 8 code +#pragma unroll + for (int k = 0; k < BMM1_XMMAS_K; ++k) { + if (elect_one) + utcmma_asmem_bsmem_h_cta1(utcmma_smem_desc_Q.desc, + utcmma_smem_desc_K.desc, tmem_acc, + utcmma_instruction_desc_8, (k > 0)); + if ((k % 4) == ((BMM1_XMMAS_K - 1) % 4)) { + utcmma_smem_desc_Q.add_smem_offset<-BYTES_PER_MMA_K *( + (BMM1_XMMAS_K - 1) % 4)>(); + utcmma_smem_desc_Q.add_smem_offset<128 * BMM1_TILE_M>(); + } else { + utcmma_smem_desc_Q.add_smem_offset(); + } + if ((k % 4) == ((BMM1_XMMAS_K - 1) % 4)) { + utcmma_smem_desc_K.add_smem_offset<-BYTES_PER_MMA_K *( + (BMM1_XMMAS_K - 1) % 4)>(); + utcmma_smem_desc_K + .add_smem_offset<128 * BMM1_TILE_N / CTA_MMA_1>(); + } else { + utcmma_smem_desc_K.add_smem_offset(); + } + } + utcmma_smem_desc_Q.add_smem_offset<-128 * BMM1_TILE_M * + FORT_MAX(BMM1_XMMAS_K / 4, 1)>(); + utcmma_smem_desc_K.add_smem_offset<-128 * BMM1_TILE_N / CTA_MMA_1 * + FORT_MAX(BMM1_XMMAS_K / 4, 1)>(); + // change the smem buffer for K + utcmma_smem_desc_K + .increment_smem_buffer(); + // change the smem buffer for Q + utcmma_smem_desc_Q + .increment_smem_buffer(); + const int kv_stage = kv_mbar_state.index(); + if (elect_one) + umma_arrive(cast_smem_ptr_to_uint( + &shared_storage.tma_k_empty_mbar[kv_stage])); + } + + if (elect_one) { + umma_arrive( + cast_smem_ptr_to_uint(&shared_storage.bmm1_tile1_done_mbar[0])); + } // elect_one + } + if (kv_loop_left_bound < kv_loop_right_bound) { + if (elect_one) { + umma_arrive( + cast_smem_ptr_to_uint(&shared_storage.tma_q_empty_mbar[1])); + umma_arrive( + cast_smem_ptr_to_uint(&shared_storage.tma_q_empty_mbar[0])); + } + const int kv_stage = kv_mbar_state.index(); + const uint32_t kv_state = kv_mbar_state.phase(); + ++kv_mbar_state; + + wait_barrier( + cast_smem_ptr_to_uint(&shared_storage.tma_v_full_mbar[kv_stage]), + kv_state); + + // S * V matmul for sub-tile 0 + { + uint32_t bmm2_ready_mbar = + cast_smem_ptr_to_uint(&shared_storage.bmm2_tile0_ready_mbar[0]); + uint32_t tmem_O = tmem_O_0; + uint32_t tmem_S = tmem_S_bmm2_0; + uint32_t scaleC = kv_loop_right_bound > kv_loop_left_bound + 1; + + // mma_op 17 code +#pragma unroll + for (int k = 0; k < BMM2_XMMAS_K; ++k) { + if (k % (BMM2_XMMAS_K / (BMM1_TILE_N / 64)) == 0) { + wait_barrier(bmm2_ready_mbar + +)KERNEL" + R"KERNEL( + (k / (BMM2_XMMAS_K / (BMM1_TILE_N / 64))) * 8, + bmm_mbar_state); + } + if (elect_one) + utcmma_atmem_bsmem_h_cta1( + tmem_S + k * (UTCMMA_TILE_K / (4 / BYTES_PER_ELEMENT)), + utcmma_smem_desc_V.desc, tmem_O, utcmma_instruction_desc_17, + scaleC || k > 0); + if (k == BMM2_XMMAS_K - 1) { + utcmma_smem_desc_V + .add_smem_offset<-UTCMMA_TILE_K * 128 * (BMM2_XMMAS_K - 1)>(); + } else { + utcmma_smem_desc_V.add_smem_offset(); + } + } + if (elect_one) + umma_arrive( + cast_smem_ptr_to_uint(&shared_storage.bmm2_tile0_done_mbar[0])); + } + // S * V matmul for sub-tile 1 + { + uint32_t bmm2_ready_mbar = + cast_smem_ptr_to_uint(&shared_storage.bmm2_tile1_ready_mbar[0]); + uint32_t tmem_O = tmem_O_1; + uint32_t tmem_S = tmem_S_bmm2_1; + uint32_t scaleC = kv_loop_right_bound > kv_loop_left_bound + 1; + + // mma_op 17 code +#pragma unroll + for (int k = 0; k < BMM2_XMMAS_K; ++k) { + if (k % (BMM2_XMMAS_K / (BMM1_TILE_N / 64)) == 0) { + wait_barrier(bmm2_ready_mbar + + (k / (BMM2_XMMAS_K / (BMM1_TILE_N / 64))) * 8, + bmm_mbar_state); + } + if (elect_one) + utcmma_atmem_bsmem_h_cta1( + tmem_S + k * (UTCMMA_TILE_K / (4 / BYTES_PER_ELEMENT)), + utcmma_smem_desc_V.desc, tmem_O, utcmma_instruction_desc_17, + scaleC || k > 0); + if (k == BMM2_XMMAS_K - 1) { + utcmma_smem_desc_V + .add_smem_offset<-UTCMMA_TILE_K * 128 * (BMM2_XMMAS_K - 1)>(); + } else { + utcmma_smem_desc_V.add_smem_offset(); + } + } + if (elect_one) + umma_arrive( + cast_smem_ptr_to_uint(&shared_storage.bmm2_tile1_done_mbar[0])); + if (elect_one) + umma_arrive(cast_smem_ptr_to_uint( + &shared_storage.tma_v_empty_mbar[kv_stage])); + // Change the buffer for V + utcmma_smem_desc_V + .increment_smem_buffer(); + } + bmm_mbar_state ^= 1; + } else { + wait_barrier( + cast_smem_ptr_to_uint(&shared_storage.empty_mainloop_mbar[0]), + empty_mainloop_mbar_state); + empty_mainloop_mbar_state ^= 1; + if (elect_one) { + umma_arrive( + cast_smem_ptr_to_uint(&shared_storage.bmm2_tile0_done_mbar[0])); + umma_arrive( + cast_smem_ptr_to_uint(&shared_storage.bmm2_tile1_done_mbar[0])); + } + } + final_stat_mbar_state ^= 1; + __syncwarp(); + + // Get next tile id + wait_barrier(cast_smem_ptr_to_uint( + &shared_storage.scheduler_mbar[sched_state.index()]), + sched_state.phase()); + uint32_t tile_id_smem_int_ptr = cast_smem_ptr_to_uint( + &shared_storage.tile_id[sched_state.index() * 8]); + lds_128(tile_id_and_coord, tile_id_smem_int_ptr); + ++sched_state; + + uint32_t tile_idx = tile_id_and_coord[0]; + uint32_t bidhb, l2_mod; + uint32_t block = 0, bidhb_residual = 0; + + fastDivMod(l2_major_divmod, tile_idx, bidhb, l2_mod); + if (bidhb < num_hb_quotient) { + { + fastDivMod(l2_minor_divmod, l2_mod, block, bidhb_residual); + } + } else { + { + fastDivMod(l2_minor_residual_divmod, l2_mod, block, bidhb_residual); + } + } + + int bidhb_actual = bidhb * l2_minor_divmod.val + bidhb_residual; + uint32_t batch_idx, head_idx; + fastDivMod(num_head_divmod, bidhb_actual, batch_idx, head_idx); + block = num_block - 1 - block; + + lean_tile_id = tile_id_and_coord[0]; + + blocked_row_coord = block + 0; + head_coord_from_grid_1 = (head_idx) + 0; + batch_coord_1 = batch_idx; + is_valid_tile = tile_id_and_coord[2] & 1; + + head_coord_1 = head_coord_from_grid_1; + head_coord_k_1 = head_coord_1 / attnDesc.q_heads_per_k; + head_coord_v_1 = head_coord_1 / attnDesc.q_heads_per_v; + + q_row_coord = (blocked_row_coord * Tiles_Q * BMM1_TILE_M); + int2 kv_loop_bounds = compute_kv_loop_bounds( + q_row_coord, BMM1_TILE_M * Tiles_Q * CGA_M_1, TILE_N_1, + actual_seqlen_kv_1, actual_seqlen_q_1, shift_right_bound, left_bound); + kv_loop_left_bound = kv_loop_bounds.x; + kv_loop_right_bound = kv_loop_bounds.y; + if (actual_seqlen_q_1 == 0 || q_row_coord >= actual_seqlen_q_1) { + kv_loop_right_bound = kv_loop_left_bound; + } + } + wait_barrier( + cast_smem_ptr_to_uint(&shared_storage.tmem_dealloc_smem_bar[0]), 0); + tmem_free_1sm(num_columns_per_tmem, shared_storage.tmem_base_ptrs); + } + + // thread_specialize_op 24 code + else if (wid == SoftmaxWarps + CorrectionWarps + 1) { + // TMALDG Q, K, V + reg_dealloc(); + + // Memory barrier states + uint32_t epilogue_state = 1; + PipelineState q_mbar_state(0, 1); // index, state + PipelineState kv_mbar_state(0, 1); // index, state + PipelineState<2> sts_page_buffer_state(0, 0); // index, state + PipelineState<2> lds_page_buffer_state(0, 0); // index, state + +// Persistent loop over tiles +#pragma unroll 1 + while (is_valid_tile) { + if (elect_one) { + arrive_barrier(cast_smem_ptr_to_uint( + &(shared_storage.read_tile_id_done_mbar[sched_state.index()]))); + } + + // global_load_shared_store_op 4 decls + + // global_load_shared_store_op 7 decls + + // global_load_shared_store_op 16 decls + if (kv_loop_left_bound < kv_loop_right_bound) { + int kv_loop = kv_loop_left_bound; + + const int q_stage = q_mbar_state.index(); + const int q_phase = q_mbar_state.phase(); + wait_barrier( + cast_smem_ptr_to_uint(&shared_storage.tma_q_empty_mbar[q_stage]), + q_phase); + ++q_mbar_state; + + // Load Q sub-tile 0 + if (true) { + smem_bar_set_transaction_count( + cast_smem_ptr_to_uint(&shared_storage.tma_q_full_mbar[q_stage]), + qTmaTransactionBytes, elect_one); + } + { + uint32_t local_smem_q = cast_smem_ptr_to_uint(shared_storage.smem_Q + + q_stage * qBufferElems); + uint32_t local_smem_bar_tma_q = + cast_smem_ptr_to_uint(&shared_storage.tma_q_full_mbar[q_stage]); + const uint32_t row_coord = q_row_coord; + + // global_load_shared_store_op 4 code +#pragma unroll + for (int i = 0; i < TILE_O_1 * BYTES_PER_ELEMENT_1; i += 128) { + utmaldg_4d_tiled(&tma_tensor_3, local_smem_q + i * TILE_M_1, + local_smem_bar_tma_q, i / BYTES_PER_ELEMENT_1, + row_coord, head_coord_1 + 0, batch_coord_1, + elect_one); + } + } + + // Load K + const int k_stage = kv_mbar_state.index(); + const int k_phase = kv_mbar_state.phase(); + wait_barrier( + cast_smem_ptr_to_uint(&shared_storage.tma_k_empty_mbar[k_stage]), + k_phase); + + if (true) { + smem_bar_set_transaction_count( + cast_smem_ptr_to_uint(&shared_storage.tma_k_full_mbar[k_stage]), + kTmaTransactionBytes, elect_one); + } + { + const int lds_page_buffer_state_idx = lds_page_buffer_state.index(); + uint32_t local_smem_k = cast_smem_ptr_to_uint(shared_storage.smem_K + + k_stage * kBufferElems); + uint32_t local_smem_bar_tma_k = + cast_smem_ptr_to_uint(&shared_storage.tma_k_full_mbar[k_stage]); + const uint32_t p_col_coord = kv_loop * TILE_N_1; + + // global_load_shared_store_op 7 code +#pragma unroll + for (int i = 0; i < TILE_O_1 * BYTES_PER_ELEMENT_1; i += 128) { + utmaldg_4d_tiled(&tma_tensor_2, local_smem_k + i * TILE_N_1, + local_smem_bar_tma_k, i / BYTES_PER_ELEMENT_1, + p_col_coord, head_coord_k_1, batch_coord_1, + elect_one); + } + } + } // if (kv_loop_left_bound < kv_loop_right_bound) + + if (kv_loop_left_bound < kv_loop_right_bound) { + const int q_stage = q_mbar_state.index(); + const int q_phase = q_mbar_state.phase(); + wait_barrier( + cast_smem_ptr_to_uint(&shared_storage.tma_q_empty_mbar[q_stage]), + q_phase); + ++q_mbar_state; + + // Load Q sub-tile 1 + if (true) { + smem_bar_set_transaction_count( + cast_smem_ptr_to_uint(&shared_storage.tma_q_full_mbar[q_stage]), + qTmaTransactionBytes, elect_one); + } + { + uint32_t local_smem_q = cast_smem_ptr_to_uint(shared_storage.smem_Q + + q_stage * qBufferElems); + uint32_t local_smem_bar_tma_q = + cast_smem_ptr_to_uint(&shared_storage.tma_q_full_mbar[q_stage]); + const uint32_t row_coord = q_row_coord + TILE_M_1; + + // global_load_shared_store_op 4 code +#pragma unroll + for (int i = 0; i < TILE_O_1 * BYTES_PER_ELEMENT_1; i += 128) { + utmaldg_4d_tiled(&tma_tensor_3, local_smem_q + i * TILE_M_1, + local_smem_bar_tma_q, i / BYTES_PER_ELEMENT_1, + row_coord, head_coord_1 + 0, batch_coord_1, + elect_one); + } + } + } // if (kv_loop_left_bound < kv_loop_right_bound) + wait_barrier(cast_smem_ptr_to_uint(&shared_storage.epilogue_done_mbar[0]), + epilogue_state); + epilogue_state ^= 1; + if (kv_loop_left_bound < kv_loop_right_bound) { + int kv_loop = kv_loop_left_bound; + // Load V + const int v_stage = kv_mbar_state.index(); + const int v_phase = kv_mbar_state.phase(); + wait_barrier( + cast_smem_ptr_to_uint(&shared_storage.tma_v_empty_mbar[v_stage]), + v_phase); + ++kv_mbar_state; + + if (true) { + smem_bar_set_transaction_count( + cast_smem_ptr_to_uint(&shared_storage.tma_v_full_mbar[v_stage]), + vTmaTransactionBytes, elect_one); + } + { + const int lds_page_buffer_state_idx = lds_page_buffer_state.index(); + uint32_t local_smem_v = cast_smem_ptr_to_uint(shared_storage.smem_V + + v_stage * vBufferElems); + uint32_t local_smem_bar_tma_v = + cast_smem_ptr_to_uint(&shared_storage.tma_v_full_mbar[v_stage]); + const uint32_t p_col_coord = kv_loop * TILE_N_1; + + // global_load_shared_store_op 16 code +#pragma unroll + for (int i = 0; i < TILE_O_1 * BYTES_PER_ELEMENT_1; + i += 128 * CTA_MMA_1) { + utmaldg_4d_tiled(&tma_tensor_1, local_smem_v + i * TILE_N_1, + local_smem_bar_tma_v, 0 + i / BYTES_PER_ELEMENT_1, + p_col_coord, head_coord_v_1, batch_coord_1, + elect_one); + } + } + } // if (kv_loop_left_bound < kv_loop_right_bound) + arrive_barrier( + cast_smem_ptr_to_uint(&shared_storage.tmaldg_tile_started_mbar[0])); + +// Mainloop to load K and V +#pragma unroll 1 + for (int kv_loop = kv_loop_left_bound + 1; kv_loop < kv_loop_right_bound; + kv_loop += 1) { + + // Load K + const int k_stage = kv_mbar_state.index(); + const int k_phase = kv_mbar_state.phase(); + wait_barrier( + cast_smem_ptr_to_uint(&shared_storage.tma_k_empty_mbar[k_stage]), + k_phase); + +)KERNEL" + R"KERNEL( + if (true) { + smem_bar_set_transaction_count( + cast_smem_ptr_to_uint(&shared_storage.tma_k_full_mbar[k_stage]), + kTmaTransactionBytes, elect_one); + } + { + const int lds_page_buffer_state_idx = lds_page_buffer_state.index(); + uint32_t local_smem_k = cast_smem_ptr_to_uint(shared_storage.smem_K + + k_stage * kBufferElems); + uint32_t local_smem_bar_tma_k = + cast_smem_ptr_to_uint(&shared_storage.tma_k_full_mbar[k_stage]); + const uint32_t p_col_coord = kv_loop * TILE_N_1; + + // global_load_shared_store_op 7 code +#pragma unroll + for (int i = 0; i < TILE_O_1 * BYTES_PER_ELEMENT_1; i += 128) { + utmaldg_4d_tiled(&tma_tensor_2, local_smem_k + i * TILE_N_1, + local_smem_bar_tma_k, i / BYTES_PER_ELEMENT_1, + p_col_coord, head_coord_k_1, batch_coord_1, + elect_one); + } + } + if (kv_loop + 1 < kv_loop_right_bound) { + } + + // Load V + const int v_stage = kv_mbar_state.index(); + const int v_phase = kv_mbar_state.phase(); + wait_barrier( + cast_smem_ptr_to_uint(&shared_storage.tma_v_empty_mbar[v_stage]), + v_phase); + ++kv_mbar_state; + + if (true) { + smem_bar_set_transaction_count( + cast_smem_ptr_to_uint(&shared_storage.tma_v_full_mbar[v_stage]), + vTmaTransactionBytes, elect_one); + } + { + const int lds_page_buffer_state_idx = lds_page_buffer_state.index(); + uint32_t local_smem_v = cast_smem_ptr_to_uint(shared_storage.smem_V + + v_stage * vBufferElems); + uint32_t local_smem_bar_tma_v = + cast_smem_ptr_to_uint(&shared_storage.tma_v_full_mbar[v_stage]); + const uint32_t p_col_coord = kv_loop * TILE_N_1; + + // global_load_shared_store_op 16 code +#pragma unroll + for (int i = 0; i < TILE_O_1 * BYTES_PER_ELEMENT_1; + i += 128 * CTA_MMA_1) { + utmaldg_4d_tiled(&tma_tensor_1, local_smem_v + i * TILE_N_1, + local_smem_bar_tma_v, 0 + i / BYTES_PER_ELEMENT_1, + p_col_coord, head_coord_v_1, batch_coord_1, + elect_one); + } + } + + ++lds_page_buffer_state; + } + if (kv_loop_left_bound < kv_loop_right_bound) { + ++lds_page_buffer_state; + ++sts_page_buffer_state; + } + __syncwarp(); + // Get next tile id + wait_barrier(cast_smem_ptr_to_uint( + &shared_storage.scheduler_mbar[sched_state.index()]), + sched_state.phase()); + uint32_t tile_id_smem_int_ptr = cast_smem_ptr_to_uint( + &shared_storage.tile_id[sched_state.index() * 8]); + lds_128(tile_id_and_coord, tile_id_smem_int_ptr); + ++sched_state; + + uint32_t tile_idx = tile_id_and_coord[0]; + uint32_t bidhb, l2_mod; + uint32_t block = 0, bidhb_residual = 0; + + fastDivMod(l2_major_divmod, tile_idx, bidhb, l2_mod); + if (bidhb < num_hb_quotient) { + { + fastDivMod(l2_minor_divmod, l2_mod, block, bidhb_residual); + } + } else { + { + fastDivMod(l2_minor_residual_divmod, l2_mod, block, bidhb_residual); + } + } + + int bidhb_actual = bidhb * l2_minor_divmod.val + bidhb_residual; + uint32_t batch_idx, head_idx; + fastDivMod(num_head_divmod, bidhb_actual, batch_idx, head_idx); + block = num_block - 1 - block; + + lean_tile_id = tile_id_and_coord[0]; + + blocked_row_coord = block + 0; + head_coord_from_grid_1 = (head_idx) + 0; + batch_coord_1 = batch_idx; + is_valid_tile = tile_id_and_coord[2] & 1; + + head_coord_1 = head_coord_from_grid_1; + head_coord_k_1 = head_coord_1 / attnDesc.q_heads_per_k; + head_coord_v_1 = head_coord_1 / attnDesc.q_heads_per_v; + + q_row_coord = (blocked_row_coord * Tiles_Q * BMM1_TILE_M); + int2 kv_loop_bounds = compute_kv_loop_bounds( + q_row_coord, BMM1_TILE_M * Tiles_Q * CGA_M_1, TILE_N_1, + actual_seqlen_kv_1, actual_seqlen_q_1, shift_right_bound, left_bound); + kv_loop_left_bound = kv_loop_bounds.x; + kv_loop_right_bound = kv_loop_bounds.y; + if (actual_seqlen_q_1 == 0 || q_row_coord >= actual_seqlen_q_1) { + kv_loop_right_bound = kv_loop_left_bound; + } + } + } + + // thread_specialize_op 25 code + else if (wid == SoftmaxWarps + CorrectionWarps + 2) { + // TMASTG (epilogue) + reg_dealloc(); + + ElementO *smem_O_0 = shared_storage.smem_O; + ElementO *smem_O_1 = shared_storage.smem_V; + // Memory barrier states + uint32_t epilogue_state = 0; + +// Persistent loop over tiles +#pragma unroll 1 + while (is_valid_tile) { + if (elect_one) { + arrive_barrier(cast_smem_ptr_to_uint( + &(shared_storage.read_tile_id_done_mbar[sched_state.index()]))); + } + + wait_barrier( + cast_smem_ptr_to_uint(&shared_storage.tmaldg_tile_started_mbar[0]), + epilogue_state); + + // global_store_op 20 decls + +// TMASTG for sub-tile 0 +#pragma unroll + for (int i = 0; i < BMM2_TILE_N * sizeof(ElementO); i += 128) { + wait_barrier( + cast_smem_ptr_to_uint(&shared_storage.tma_o_0_full_mbar[i / 128]), + epilogue_state); + const uint32_t row_coord = q_row_coord; + uint32_t smem_loc = cast_smem_ptr_to_uint(smem_O_0) + i * BMM2_TILE_M; + if (actual_seqlen_q_1 > 0) { + + // global_store_op 20 code + utmastg_4d_tiled(&tma_tensor_5, smem_loc, 0 + i / sizeof(ElementO), + row_coord, head_coord_1 + 0, batch_coord_1); + } + } + + tmastg_arrive(); + +// TMASTG for sub-tile 1 +#pragma unroll + for (int i = 0; i < BMM2_TILE_N * sizeof(ElementO); i += 128) { + wait_barrier( + cast_smem_ptr_to_uint(&shared_storage.tma_o_1_full_mbar[i / 128]), + epilogue_state); + const uint32_t row_coord = q_row_coord + TILE_M_1; + uint32_t smem_loc = cast_smem_ptr_to_uint(smem_O_1) + i * BMM2_TILE_M; + if (actual_seqlen_q_1 > 0) { + + // global_store_op 20 code + utmastg_4d_tiled(&tma_tensor_5, smem_loc, 0 + i / sizeof(ElementO), + row_coord, head_coord_1 + 0, batch_coord_1); + } + } + tmastg_arrive(); + + tmastg_wait_count<1>(); + arrive_barrier( + cast_smem_ptr_to_uint(&(shared_storage.tma_o_0_empty_mbar[0]))); + + tmastg_wait_count<0>(); + arrive_barrier( + cast_smem_ptr_to_uint(&(shared_storage.epilogue_done_mbar[0]))); + + epilogue_state ^= 1; + + // Get next tile id + wait_barrier(cast_smem_ptr_to_uint( + &shared_storage.scheduler_mbar[sched_state.index()]), + sched_state.phase()); + uint32_t tile_id_smem_int_ptr = cast_smem_ptr_to_uint( + &shared_storage.tile_id[sched_state.index() * 8]); + lds_128(tile_id_and_coord, tile_id_smem_int_ptr); + ++sched_state; + + uint32_t tile_idx = tile_id_and_coord[0]; + uint32_t bidhb, l2_mod; + uint32_t block = 0, bidhb_residual = 0; + + fastDivMod(l2_major_divmod, tile_idx, bidhb, l2_mod); + if (bidhb < num_hb_quotient) { + { + fastDivMod(l2_minor_divmod, l2_mod, block, bidhb_residual); + } + } else { + { + fastDivMod(l2_minor_residual_divmod, l2_mod, block, bidhb_residual); + } + } + + int bidhb_actual = bidhb * l2_minor_divmod.val + bidhb_residual; + uint32_t batch_idx, head_idx; + fastDivMod(num_head_divmod, bidhb_actual, batch_idx, head_idx); + block = num_block - 1 - block; + + lean_tile_id = tile_id_and_coord[0]; + + blocked_row_coord = block + 0; + head_coord_from_grid_1 = (head_idx) + 0; + batch_coord_1 = batch_idx; + is_valid_tile = tile_id_and_coord[2] & 1; + + head_coord_1 = head_coord_from_grid_1; + head_coord_k_1 = head_coord_1 / attnDesc.q_heads_per_k; + head_coord_v_1 = head_coord_1 / attnDesc.q_heads_per_v; + + q_row_coord = (blocked_row_coord * Tiles_Q * BMM1_TILE_M); + int2 kv_loop_bounds = compute_kv_loop_bounds( + q_row_coord, BMM1_TILE_M * Tiles_Q * CGA_M_1, TILE_N_1, + actual_seqlen_kv_1, actual_seqlen_q_1, shift_right_bound, left_bound); + kv_loop_left_bound = kv_loop_bounds.x; + kv_loop_right_bound = kv_loop_bounds.y; + if (actual_seqlen_q_1 == 0 || q_row_coord >= actual_seqlen_q_1) { + kv_loop_right_bound = kv_loop_left_bound; + } + } // end of persistent loop + } + + // thread_specialize_op 26 code + else if (wid == SoftmaxWarps + CorrectionWarps + 3) { + // Scheduler + reg_dealloc(); + +#pragma unroll 1 + while (is_valid_tile) { + wait_barrier( + cast_smem_ptr_to_uint( + &shared_storage.read_tile_id_done_mbar[sched_state.index()]), + sched_state.phase()); + uint32_t tile_id_smem_int_ptr = cast_smem_ptr_to_uint( + &shared_storage.tile_id[sched_state.index() * 8]); + + smem_bar_set_transaction_count( + cast_smem_ptr_to_uint( + &shared_storage.scheduler_mbar[sched_state.index()]), + 16, elect_one); + + if (elect_one) { + get_next_block_id( + tile_id_smem_int_ptr, + cast_smem_ptr_to_uint( + &shared_storage.scheduler_mbar[sched_state.index()])); + } + __syncwarp(); + + wait_barrier(cast_smem_ptr_to_uint( + &shared_storage.scheduler_mbar[sched_state.index()]), + sched_state.phase()); + lds_128(tile_id_and_coord, tile_id_smem_int_ptr); + ++sched_state; + + is_valid_tile = tile_id_and_coord[2] & 1; + } + } +} +)KERNEL"; +inline constexpr size_t sm100_d128_fprop_source_len = sizeof(sm100_d128_fprop_source) - 1; + +inline constexpr const char sm100_d128_fprop_flags[] = R"FLAGS( +--gpu-architecture=sm_100a +--std=c++17 +-w +--define-macro=__CUDACC_RTC__ +-default-device +--use_fast_math +-Xptxas=-maxrregcount=128 +-DCUDA_ENABLE_TENSOR_MEMORY_INTRINSICS=1 +-DCUDA_ENABLE_VIRTCOUNT_INTRINSICS=1 +-DCUDA_ENABLE_TMEM_MANAGEMENT_INTRINSICS=1 +-DCUDA_BLACKWELL_TMA_SWIZZLE_ENABLED=1 +-DCUDA_ENABLE_FLEXIBLE_CLUSTER=1 +-DCUDA_PTX_TCMMA_V2_SUPPORTED=1 +-DCUDA_PTX_TMEM_MANAGEMENT_SUPPORTED=1 +-DCUDA_ENABLE_CLUSTER_MMA_INTRINSICS=1 +-DCUTLASS_ENABLE_TENSOR_CORE_MMA=1 +-DCUTLASS_ENABLE_EXTENDED_PTX=1 +-DCUTLASS_ENABLE_INTERNAL_NVVM=1 +-DCUTLASS_CUDA_INTERNAL_L2_PREFETCH_ENABLED=1 +-DCUTLASS_CUDA_RP2RP_ENABLED=1 +-DCUTLASS_ENABLE_COMPILER_KNOBS=1 +-DCUTLASS_TEST_LEVEL=0 +-DCUTLASS_TEST_ENABLE_CACHED_RESULTS=1 +-DCUTLASS_CONV_UNIT_TEST_RIGOROUS_SIZE_ENABLED=1 +-DCUTLASS_DEBUG_TRACE_LEVEL=0 +-DCUTLASS_VERSIONS_GENERATED +)FLAGS"; +inline constexpr size_t sm100_d128_fprop_flags_len = sizeof(sm100_d128_fprop_flags) - 1; + +} // namespace generated +} // namespace experimental +} // namespace cudnn_frontend +#if defined(_MSC_VER) +#pragma warning(pop) +#endif +#if defined(__clang__) +#pragma clang diagnostic pop +#endif \ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/generated/sdpa/sm100/prefill/full_seqlens/d64_fprop_kernel.h b/deps/cudnn-frontend/include/cudnn_frontend/generated/sdpa/sm100/prefill/full_seqlens/d64_fprop_kernel.h new file mode 100644 index 0000000..33b9ef2 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/generated/sdpa/sm100/prefill/full_seqlens/d64_fprop_kernel.h @@ -0,0 +1,3481 @@ +// Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: MIT +#pragma once +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Woverlength-strings" +#endif +#if defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable : 4068) +#endif +namespace cudnn_frontend { +namespace experimental { +namespace generated { +inline constexpr const char sm100_d64_fprop_source[] = + R"KERNEL( + +// receive_op 0 includes +typedef unsigned long long uint64_t; +typedef unsigned int uint32_t; +typedef unsigned short uint16_t; +typedef unsigned char uint8_t; +typedef long long int64_t; +typedef int int32_t; +typedef short int16_t; +typedef signed char int8_t; +typedef unsigned int r32; + +struct CUtensorMap { + alignas(64) uint64_t opaque[16]; +}; + +#define __FLT_MIN__ 1.17549435082228750796873653722224568e-38F +#define __FLT_MAX__ 3.40282346638528859811704183484516925e+38F + +#define CUDACC_VERSION __CUDACC_VER_MAJOR__ * 10 + __CUDACC_VER_MINOR__ + +namespace fort { +static const uint64_t AMPERE_MEM_DESC_DEFAULT = uint64_t(0x1000000000000000ul); +static const uint64_t MEM_DESC_DEFAULT = AMPERE_MEM_DESC_DEFAULT; +static const uint32_t MMA_PEER_BIT_MASK = 0xFEFFFFFF; + +#define NAN __int_as_float(0x7fffffff) +#define POS_INFINITY __int_as_float(0x7f800000) +#define NEG_INFINITY __int_as_float(0xff800000) + +typedef uint16_t bfloat16_t; + +#define FORT_MIN(a, b) ((a) < (b) ? (a) : (b)) +#define FORT_MAX(a, b) ((a) > (b) ? (a) : (b)) +#define FORT_DIV_UP(a, b) (((a) + (b) - 1) / (b)) +#define FORT_ROUND_UP(a, b) ((((a) + (b) - 1) / (b)) * (b)) + +typedef struct tensor_descriptor { + static const int MAX_DIMS = 12; + + int64_t num_dims; + int64_t dims[MAX_DIMS]; + int64_t strides[MAX_DIMS]; +} tensor_descriptor; + +typedef struct FastDivisor { + uint32_t val, shr, mul; +} FastDivisor_t; + +inline __device__ void fastDivMod(const FastDivisor_t &d, uint32_t val, + uint32_t &div, uint32_t &mod) { + div = __umulhi((uint32_t)2 * val, d.mul) >> d.shr; + mod = val - div * d.val; +} + +inline __device__ char *get_smem_loc_epilogue_swizzle_128b( + char *smem_addr, int local_block_id, int tid, int local_row, int column, + size_t element_size, int block_size, int row_per_tile) { + if (element_size * block_size == 256) { + // Case 1: 2 * swizzle tile size = block size -> half block per swizzle tile + int swizzle_col = local_row ^ (column % 8); + + // (local_block_id * 2 + (column / 8)) * row_per_tile * 128: gives offset of + // corresponding swizzle tile tid * 128: tid gives number of loads in block + // swizzle_col * 16B: swizzle_col gives transformed column + return smem_addr + + (local_block_id * 2 + (column / 8)) * row_per_tile * 128 + + tid * 128 + swizzle_col * 16; + } else if (element_size * block_size == 128) { + // Case 2: swizzle tile size = block size -> one block per swizzle tile + int swizzle_col = local_row ^ (column % 8); + + // local_block_id * row_per_tile * 128: gives offset of corresponding + // swizzle tile tid * 128: tid gives number of loads in block swizzle_col * + // 16B: swizzle_col gives transformed column + return smem_addr + local_block_id * row_per_tile * 128 + tid * 128 + + swizzle_col * 16; + } else { + // Case 3: swizzle tile size / 2 = block size -> two blocks per swizzle tile + int swizzle_col = local_row ^ ((column + local_block_id * 4) % 8); + + // (local_block_id / 2) * row_per_tile * 128: gives offset of corresponding + // swizzle tile tid * 128: tid gives number of loads in block swizzle_col * + // 16B: swizzle_col gives transformed column + return smem_addr + (local_block_id / 2) * row_per_tile * 128 + tid * 128 + + swizzle_col * 16; + } +} + +inline __device__ static void get_next_block_id(uint32_t result, + uint32_t mbar) { +#if __CUDA_ARCH__ >= 1200 && __CUDA_ARCH__ < 1300 + asm volatile("{clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::" + "complete_tx::bytes.b128 [%0], [%1];}\n\t" ::"r"(result), + "r"(mbar)); +#else + asm volatile( + "{clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_" + "tx::bytes.multicast::cluster::all.b128 [%0], [%1];}\n\t" ::"r"(result), + "r"(mbar)); +#endif +} + +inline __device__ float2 ffma2(const float2 &a, const float2 &b, + const float2 &c) { + uint64_t d; + asm volatile("fma.rn.f32x2 %0, %1, %2, %3;\n" + : "=l"(d) + : "l"(reinterpret_cast(a)), + "l"(reinterpret_cast(b)), + "l"(reinterpret_cast(c))); + return reinterpret_cast(d); +} + +inline __device__ float2 fadd2(const float2 &a, const float2 &b) { + uint64_t c; + asm volatile("add.f32x2 %0, %1, %2;\n" + : "=l"(c) + : "l"(reinterpret_cast(a)), + "l"(reinterpret_cast(b))); + return reinterpret_cast(c); +} + +inline __device__ float2 fmul2(const float2 &a, const float2 &b) { + uint64_t c; + asm volatile("mul.f32x2 %0, %1, %2;\n" + : "=l"(c) + : "l"(reinterpret_cast(a)), + "l"(reinterpret_cast(b))); + return reinterpret_cast(c); +} + +inline __device__ float fmax3(float a, float b, float c) { +#if (__CUDACC_VER_MAJOR__ >= 13) + float d; + asm volatile("max.ftz.f32 %0, %1, %2, %3;\n" + : "=f"(d) + : "f"((a)), "f"((b)), "f"((c))); + return d; +#else + #error "This kernel is not supported for CUDA toolkit version < 13.0" + return 0.0f; +#endif +} + +inline __device__ float row_max_reduction_128_elems(uint32_t reg[128]) { + float tmp_max_0[42]; + float tmp_max_1[14]; + float tmp_max_2[5]; + float tmp_max_3[2]; + +#pragma unroll + for (int i = 0; i < 126; i += 3) { + tmp_max_0[i / 3] = fmax3(reinterpret_cast(reg[i + 0]), + reinterpret_cast(reg[i + 1]), + reinterpret_cast(reg[i + 2])); + } +#pragma unroll + for (int i = 0; i < 42; i += 3) { + tmp_max_1[i / 3] = + fmax3(tmp_max_0[i + 0], tmp_max_0[i + 1], tmp_max_0[i + 2]); + } +#pragma unroll + for (int i = 0; i < 12; i += 3) { + tmp_max_2[i / 3] = + fmax3(tmp_max_1[i + 0], tmp_max_1[i + 1], tmp_max_1[i + 2]); + } + tmp_max_2[4] = + fmax3(tmp_max_1[12], tmp_max_1[13], reinterpret_cast(reg[126])); + tmp_max_3[0] = fmax3(tmp_max_2[0], tmp_max_2[1], tmp_max_2[2]); + tmp_max_3[1] = + fmax3(tmp_max_2[3], tmp_max_2[4], reinterpret_cast(reg[127])); + + return fmax3(tmp_max_3[0], tmp_max_3[1], tmp_max_3[1]); +} + +#if __CUDA_ARCH__ >= 900 + +inline __device__ uint32_t elect_one_sync() { + uint32_t pred = 0; + uint32_t laneid = 0; + asm volatile("{\n" + ".reg .b32 %%rx;\n" + ".reg .pred %%px;\n" + " elect.sync %%rx|%%px, %2;\n" + "@%%px mov.s32 %1, 1;\n" + " mov.s32 %0, %%rx;\n" + "}\n" + : "+r"(laneid), "+r"(pred) + : "r"(0xFFFFFFFF)); + return pred; +} + +template inline __device__ void reg_alloc() { + // const int TARGET_REG_COUNT = 232; // Example values (use with reg_delloc's + // VALUE together): 224, 232, 208, 216 + asm volatile("setmaxnreg.inc.sync.aligned.u32 %0;\n" + : + : "n"(TARGET_REG_COUNT)); +} + +template inline __device__ void reg_dealloc() { + // const int TARGET_REG_COUNT = 40; // Exmaple values: 56, 40, 88, 72 + asm volatile("setmaxnreg.dec.sync.aligned.u32 %0;\n" + : + : "n"(TARGET_REG_COUNT)); +} + +inline __device__ void named_barrier_arrive(uint32_t BARRIER_ID, + uint32_t NUM_THREADS) { + asm volatile("bar.arrive %0, %1;" : : "r"(BARRIER_ID), "r"(NUM_THREADS)); +} + +inline __device__ void named_barrier_wait(uint32_t BARRIER_ID, + uint32_t NUM_THREADS) { + asm volatile("bar.sync %0, %1;" ::"r"(BARRIER_ID), "r"(NUM_THREADS)); +} + +inline __device__ void tmastg_arrive() { + asm volatile("cp.async.bulk.commit_group;"); +} + +template inline __device__ void tmastg_wait_count() { + asm volatile("cp.async.bulk.wait_group.read %0;" : : "n"(Count) : "memory"); +} + +#endif + +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 +inline __device__ void smem_bar_init(uint32_t smem_ptr, uint32_t thread_count) { + asm volatile("mbarrier.init.shared::cta.b64 [%0], %1;\n" ::"r"(smem_ptr), + "r"(thread_count)); +} + +inline __device__ void +smem_bar_set_transaction_count(uint32_t smem_ptr, uint32_t expected_copy_bytes, + uint32_t pred = 0) { + asm volatile("{\n\t.reg .pred p;" + " \n\tsetp.eq.u32 p, %2, 1;" + " \n\t@p mbarrier.arrive.expect_tx.shared::cta.b64 _, [%0], %1;" + " \n\t}" ::"r"(smem_ptr), + "r"(expected_copy_bytes), "r"(pred)); +} + +inline __device__ uint32_t smem_bar_peek(uint32_t smem_ptr, + uint32_t bar_phase) { + uint32_t bar_phase_out; + asm volatile("{\n\t.reg .pred P1;" + " \n\tmbarrier.try_wait.parity.shared::cta.b64 P1, [%1], %2;" + " \n\tselp.b32 %0, 1, 0, P1; \n\t}" + : "=r"(bar_phase_out) + : "r"(smem_ptr), "r"(bar_phase)); + return bar_phase_out; +} + +inline __device__ void +wait_barrier(uint32_t smem_int_ptr, + int phase_bit) // Current phase bit the barrier waiting to flip +{ + asm volatile("{\n" + ".reg .pred P1;\n" + "LAB_WAIT:\n" + "mbarrier.try_wait.parity.shared::cta.b64 P1, [%0], %1;\n" + "@P1 bra.uni DONE;\n" + "bra.uni LAB_WAIT;\n" + "DONE:\n" + "}\n" ::"r"(smem_int_ptr), + "r"(phase_bit)); +} + +// Barrier arrive on local smem +inline __device__ void arrive_barrier(uint32_t smem_addr) { + asm volatile("{\n\t" + "mbarrier.arrive.shared::cta.b64 _, [%0];\n\t" + "}" + : + : "r"(smem_addr)); +} + +#endif + +// Circular Buffer Index + Associated Phase +// Assumes only one operation possible - i.e., ++ +template struct PipelineState { +public: + static constexpr int Stages = Stages_; + + inline __device__ PipelineState(int index, int state) + : state_(state), index_(index) {} + + inline __device__ int phase() const { return state_; } + + inline __device__ int index() const { return index_; } + + inline __device__ void operator++() { + if (Stages > 1) { + ++index_; + if (index_ == Stages) { + index_ = 0; + state_ ^= 1; + } + } else { + state_ ^= 1; + } + } + + int state_; + int index_; +}; + +template +inline __device__ constexpr uint64_t build_utcmma_instruction_desc() { + return (static_cast(0) | (MAX_SHIFT_FOR_WS << 30) | + ((UTCMMA_M >> 5) << 25) | ((UTCMMA_N >> 3) << 17) | + (B_TRANSPOSE << 16) | (A_TRANSPOSE << 15) | (B_NEGATE << 14) | + (A_NEGATE << 13) | (SRC_B_TYPE << 10) | (SRC_A_TYPE << 7) | + (SPARSE_METADATA_FORMAT << 6) | (ACC_TYPE << 4) | + (SATURATE_ENABLED << 3) | (SPARSE_ENABLED << 2) | + (SPARSE_METADATA_ID2 << 0)) + << 32; +} +)KERNEL" + R"KERNEL( + +class Smem_utcmma_descriptor { +public: + inline __device__ Smem_utcmma_descriptor(uint64_t SWIZZLE_MODE, + uint64_t BASE_OFFSET, + uint64_t DESC_VERSION, + uint64_t BYTES_PER_LEADING_DIM, + uint64_t BYTES_PER_STRIDE_DIM) { + // ------------------------------------ + // Setup smem descriptor for operand A: + // ------------------------------------ + // Note 1: SWIZZLE_NONE = 0, SWIZZLE_128B = 2, SWIZZLE_64B = 4, SWIZZLE_32B + // = 6, SWIZZLE_128B_ATOM32B = 1, N/A = 3, N/A = 5, N/A = 7 + const uint64_t SWIZZLE_MODE_IN_BIT_LOCATION = SWIZZLE_MODE + << 61; // bits: 63-61 + + // Note 2: Base offset. Valid only for matrix descriptor 1, 2 or 3, 4 + const uint64_t BASE_OFFSET_IN_BIT_LOCATION = BASE_OFFSET + << 49; // bits 51-49 + + // Note 3: Descriptor version, needs to be set to 1. ??? + const uint64_t DESC_VERSION_IN_BIT_LOCATION = DESC_VERSION + << 46; // bits 48-46 + + // Note 4: Stride dimension byte offset, 16 byte aligned, 4 LSBs not + // included + const uint64_t STRIDE_DIM_BYTE_OFFSET = BYTES_PER_STRIDE_DIM >> 4; + const uint64_t STRIDE_DIM_BYTE_OFFSET_IN_BIT_LOCATION = + STRIDE_DIM_BYTE_OFFSET << 32; // bits 45-32 + + // Note 5: Leading dimension byte offset, 16 byte aligned, 4 LSBs not + // included + const uint64_t LEADING_DIM_BYTE_OFFSET = BYTES_PER_LEADING_DIM >> 4; + const uint64_t LEADING_DIM_BYTE_OFFSET_IN_BIT_LOCATION = + LEADING_DIM_BYTE_OFFSET << 16; // bits 29-16 + + desc = + (SWIZZLE_MODE_IN_BIT_LOCATION | BASE_OFFSET_IN_BIT_LOCATION | + DESC_VERSION_IN_BIT_LOCATION | STRIDE_DIM_BYTE_OFFSET_IN_BIT_LOCATION | + LEADING_DIM_BYTE_OFFSET_IN_BIT_LOCATION); + } + template + inline __device__ void set_smem(uint32_t smem) { + int2 &tmp = reinterpret_cast(desc); + tmp.x |= (static_cast(smem & 0xFFFFFF) >> 4); + + int2 &tmp_initial = reinterpret_cast(initial_desc); + tmp_initial.x = tmp.x; + + max_desc = tmp.x + (BYTES_PER_BUFFER >> 4) * (BUFFER_COUNT - 1); + } + + template + inline __device__ void increment_smem_buffer() { + int2 &tmp = reinterpret_cast(desc); + tmp.x += (tmp.x >= max_desc) ? -(BYTES_PER_BUFFER >> 4) * (BUFFER_COUNT - 1) + : (BYTES_PER_BUFFER >> 4); + } + + template + inline __device__ void stage_increment_smem_buffer(int stage) { + int2 &tmp_initial = reinterpret_cast(initial_desc); + int2 &tmp = reinterpret_cast(desc); + + tmp.x = tmp_initial.x + (BYTES_PER_BUFFER >> 4) * stage; + } + + template inline __device__ void add_smem_offset() { + int2 &tmp = reinterpret_cast(desc); + tmp.x += (BYTES_OFFSET >> 4); + } + + uint64_t initial_desc; + uint64_t desc; + uint32_t max_desc; +}; + +// Adapted from : +// cta1 utcmma with A and B from smem and C stored in tmem, scale denotes if we +// want to accumulate C or not Something like: Accumulate or overwrite C. 1: +// read C, 0: ignore C [clear accumulators] +inline __device__ void +utcmma_asmem_bsmem_h_cta1(uint64_t sdescA, uint64_t sdescB, uint32_t tmemC, + uint64_t idescE, uint32_t scaleC) { + uint32_t mask[4] = {0, 0, 0, 0}; // TODO : Figure out what this mask means, + // CUTLASS hardcodes to all zeros + asm volatile("{\n\t" + ".reg .pred p;\n\t" + "setp.ne.b32 p, %4, 0;\n\t" + "tcgen05.mma.cta_group::1.kind::f16 [%0], %1, %2, %3, {%5, %6, " + "%7, %8}, p; \n\t" + "}\n" + : + : "r"(tmemC), "l"(sdescA), "l"(sdescB), + "r"(uint32_t(idescE >> 32)), "r"(scaleC), "r"(mask[0]), + "r"(mask[1]), "r"(mask[2]), "r"(mask[3])); +} + +// cta1 utcmma with A from tmem and B from smem and C stored in tmem, scale +// denotes if we want to accumulate C or not Something like: Accumulate or +// overwrite C. 1: read C, 0: ignore C [clear accumulators] +inline __device__ void +utcmma_atmem_bsmem_h_cta1(uint32_t tmemA, uint64_t sdescB, uint32_t tmemC, + uint64_t idescE, uint32_t scaleC) { + uint32_t mask[4] = {0, 0, 0, 0}; // TODO : Figure out what this mask means, + // CUTLASS hardcodes to all zeros + asm volatile("{\n\t" + ".reg .pred p;\n\t" + "setp.ne.b32 p, %4, 0;\n\t" + "tcgen05.mma.cta_group::1.kind::f16 [%0], [%1], %2, %3, {%5, " + "%6, %7, %8}, p; \n\t" + "}\n" + : + : "r"(tmemC), "r"(tmemA), "l"(sdescB), + "r"(uint32_t(idescE >> 32)), "r"(scaleC), "r"(mask[0]), + "r"(mask[1]), "r"(mask[2]), "r"(mask[3])); +} + +// Check for the utcmma to finish +inline __device__ void umma_arrive(uint32_t bar_intptr) { + asm volatile("tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::" + "cluster.b64 [%0];" ::"r"(bar_intptr)); +} + +inline __device__ uint32_t cast_smem_ptr_to_uint(void const *const ptr) { + return static_cast(__cvta_generic_to_shared(ptr)); +} + +inline __device__ void fence_view_async_shared(void) { + asm volatile("fence.proxy.async.shared::cta;\n"); +} + +inline __device__ void sts_128(r32 dst, r32 src[4]) { + asm volatile("st.shared.v4.b32 [%0], {%1, %2, %3, %4};\n" ::"r"(dst), + "r"(src[0]), "r"(src[1]), "r"(src[2]), "r"(src[3])); +} + +inline __device__ void lds_128(r32 dst[4], r32 src) { + asm volatile("ld.shared.v4.b32 {%0, %1, %2, %3}, [%4];\n" + : "=r"(dst[0]), "=r"(dst[1]), "=r"(dst[2]), "=r"(dst[3]) + : "r"(src)); +} + +inline __device__ void lds_32(r32 dst[1], r32 src) { + asm volatile("ld.shared.b32 %0, [%1];\n" : "=r"(dst[0]) : "r"(src)); +} + +inline __device__ void stg_32(void *ptr, r32 val[1]) { + uint32_t *p = reinterpret_cast(ptr); + p[0] = val[0]; +} + +inline __device__ void stg_8(void *ptr, r32 val[1]) { + uint8_t *p = reinterpret_cast(ptr); + p[0] = reinterpret_cast(val[0]); +} + +inline __device__ void ldg_16(r32 dst[1], const void *ptr, bool pred = true) { + if (pred) { + *reinterpret_cast(dst) = + *reinterpret_cast(ptr); + } else { + *reinterpret_cast(dst) = 0; + } +} + +inline __device__ void ldg_8(r32 dst[1], const void *ptr, bool pred = true) { + if (pred) { + *reinterpret_cast(dst) = *reinterpret_cast(ptr); + } else { + *reinterpret_cast(dst) = 0; + } +} + +inline __device__ void ldg_16_reg(r32 dst[1], const void *ptr, + bool pred = true) { + if (pred) { + uint16_t tmp = *reinterpret_cast(ptr); + asm volatile("{\nmov.b32 %0, {%1, %1};\n}\n" : "=r"(dst[0]) : "h"(tmp)); + } else { + dst[0] = 0; + } +} + +inline __device__ void ldg_8_reg(r32 dst[1], const void *ptr, + bool pred = true) { + if (pred) { + uint8_t tmp_val = *reinterpret_cast(ptr); + uint8_t *tmp_ptr = reinterpret_cast(dst); + tmp_ptr[0] = tmp_ptr[1] = tmp_ptr[2] = tmp_ptr[3] = tmp_val; + } else { + dst[0] = 0; + } +} + +// Forward (non-ReLU) Activations +// Backward (non-ReLU) Activations +// Forward ReLU activations +// Relu helper function: clip the lower part +// Relu helper function: clip the upper part +// Forward ReLU: clip only lower part +// Forward ReLU: clip lower part and upper part +// Forward ReLU: clip only lower part, but add slope to lower clip +// Forward ReLU: clip lower part and upper part, but add slope to lower clip +#if __CUDA_ARCH__ >= 900 +#endif + +// optimization opportunity exists here +inline __device__ void fp32x2_to_fp16x2(r32 dst[1], const r32 src[2]) { + asm volatile("cvt.rn.f16x2.f32 %0, %1, %2;\n" + : "=r"(dst[0]) + : "r"(src[1]), "r"(src[0])); +} + +inline __device__ void fp32x2_to_fp16x2_relu(r32 dst[1], const r32 src[2]) { + asm volatile("cvt.rn.relu.f16x2.f32 %0, %1, %2;\n" + : "=r"(dst[0]) + : "r"(src[1]), "r"(src[0])); +} + +inline __device__ void fp32x2_to_bf16x2(r32 dst[1], const r32 src[2]) { + asm volatile("cvt.rn.bf16x2.f32 %0, %1, %2;\n" + : "=r"(dst[0]) + : "r"(src[1]), "r"(src[0])); +} + +// mma_pipeline_op 1 includes +typedef struct AttentionDescriptor { + // Input parameters + // b - batch + // q_h - num heads of q/dq/o/do + // k_h - num heads of k/dk + // v_h - num heads of v/dv + // s_q - max sequence length of q + // s_kv - max sequence length of kv + // d - hidden dim (head dim) + uint32_t b, q_h, k_h, v_h, s_q, s_kv, d_qk, d_v; + uint16_t q_heads_per_k, q_heads_per_v, min_q_heads_per_kv; +} AttentionDescriptor_t; + + +inline __device__ void fence_view_async_tmem_load() { + asm volatile("{\n\ttcgen05.wait::ld.sync.aligned; \n}" ::); +} + +inline __device__ void fence_view_async_tmem_store() { + asm volatile("{\n\ttcgen05.wait::st.sync.aligned; \n}" ::); +} + +inline __device__ void tmem_allocate_1sm(uint32_t num_columns, + uint32_t smem_addr) { + asm volatile( + "tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], %1;" ::"r"( + smem_addr), + "r"(num_columns)); +} + +inline __device__ void tmem_free_1sm(uint32_t num_columns, uint32_t tmem_addr) { + asm volatile( + "{\n\ttcgen05.dealloc.cta_group::1.sync.aligned.b32 %0, %1; \n\t}" ::"r"( + tmem_addr), + "r"(num_columns)); +} + +inline __device__ void tmem_release_allocation_lock_1sm() { +)KERNEL" + R"KERNEL( + asm volatile("tcgen05.relinquish_alloc_permit.cta_group::1.sync.aligned;" ::); +} + +///////////////////////////////// +/// TMEM STTM related instructions +///////////////////////////////// + +// 32 data path lanes, 32-bit pattern, repeated 128 times +// 32 data path lanes, 32-bit pattern, repeated 64 times +// 32 data path lanes, 32-bit pattern, repeated 32 times +inline __device__ void sttm_32dp32bit_x32(uint32_t dst, r32 src[32]) { + asm volatile("tcgen05.st.sync.aligned.32x32b.x32.b32" + "[%0]," + "{%1, %2, %3, %4," + "%5, %6, %7, %8," + "%9, %10, %11, %12," + "%13, %14, %15, %16," + "%17, %18, %19, %20," + "%21, %22, %23, %24," + "%25, %26, %27, %28," + "%29, %30, %31, %32};\n" + : + : "r"(dst), "r"(src[0]), "r"(src[1]), "r"(src[2]), "r"(src[3]), + "r"(src[4]), "r"(src[5]), "r"(src[6]), "r"(src[7]), + "r"(src[8]), "r"(src[9]), "r"(src[10]), "r"(src[11]), + "r"(src[12]), "r"(src[13]), "r"(src[14]), "r"(src[15]), + "r"(src[16]), "r"(src[17]), "r"(src[18]), "r"(src[19]), + "r"(src[20]), "r"(src[21]), "r"(src[22]), "r"(src[23]), + "r"(src[24]), "r"(src[25]), "r"(src[26]), "r"(src[27]), + "r"(src[28]), "r"(src[29]), "r"(src[30]), "r"(src[31])); +} + +// 32 data path lanes, 32-bit pattern, repeated 16 times +inline __device__ void sttm_32dp32bit_x16(uint32_t dst, r32 src[16]) { + asm volatile("tcgen05.st.sync.aligned.32x32b.x16.b32" + "[%0]," + "{%1, %2, %3, %4," + "%5, %6, %7, %8," + "%9, %10, %11, %12," + "%13, %14, %15, %16};\n" + : + : "r"(dst), "r"(src[0]), "r"(src[1]), "r"(src[2]), "r"(src[3]), + "r"(src[4]), "r"(src[5]), "r"(src[6]), "r"(src[7]), + "r"(src[8]), "r"(src[9]), "r"(src[10]), "r"(src[11]), + "r"(src[12]), "r"(src[13]), "r"(src[14]), "r"(src[15])); +} + +// 32 data path lanes, 32-bit pattern, repeated 8 times +inline __device__ void sttm_32dp32bit_x8(uint32_t dst, r32 src[8]) { + asm volatile("tcgen05.st.sync.aligned.32x32b.x8.b32" + "[%0]," + "{%1, %2, %3, %4," + "%5, %6, %7, %8};\n" + : + : "r"(dst), "r"(src[0]), "r"(src[1]), "r"(src[2]), "r"(src[3]), + "r"(src[4]), "r"(src[5]), "r"(src[6]), "r"(src[7])); +} + +// 32 data path lanes, 32-bit pattern, repeated 4 times +// 32 data path lanes, 32-bit pattern, repeated 2 times +inline __device__ void sttm_32dp32bit_x2(uint32_t dst, r32 src[2]) { + asm volatile("tcgen05.st.sync.aligned.32x32b.x2.b32" + "[%0]," + "{%1, %2};\n" + : + : "r"(dst), "r"(src[0]), "r"(src[1])); +} + +// 32 data path lanes, 32-bit pattern, repeated 1 times +inline __device__ void sttm_32dp32bit_x1(uint32_t dst, r32 src[1]) { + asm volatile("tcgen05.st.sync.aligned.32x32b.x1.b32" + "[%0]," + "{%1};\n" + : + : "r"(dst), "r"(src[0])); +} + +///////////////////////////////// +/// TMEM LDTM related instructions +///////////////////////////////// + +// 32 data path lanes, 32-bit pattern, repeated 128 times +// 32 data path lanes, 32-bit pattern, repeated 64 times +// 32 data path lanes, 32-bit pattern, repeated 32 times +inline __device__ void ldtm_32dp32bit_x32(r32 dst[32], uint32_t src_addr) { + asm volatile("tcgen05.ld.sync.aligned.32x32b.x32.b32" + "{%0, %1, %2, %3," + "%4, %5, %6, %7," + "%8, %9, %10, %11," + "%12, %13, %14, %15," + "%16, %17, %18, %19," + "%20, %21, %22, %23," + "%24, %25, %26, %27," + "%28, %29, %30, %31}," + "[%32];\n" + : "=r"(dst[0]), "=r"(dst[1]), "=r"(dst[2]), "=r"(dst[3]), + "=r"(dst[4]), "=r"(dst[5]), "=r"(dst[6]), "=r"(dst[7]), + "=r"(dst[8]), "=r"(dst[9]), "=r"(dst[10]), "=r"(dst[11]), + "=r"(dst[12]), "=r"(dst[13]), "=r"(dst[14]), "=r"(dst[15]), + "=r"(dst[16]), "=r"(dst[17]), "=r"(dst[18]), "=r"(dst[19]), + "=r"(dst[20]), "=r"(dst[21]), "=r"(dst[22]), "=r"(dst[23]), + "=r"(dst[24]), "=r"(dst[25]), "=r"(dst[26]), "=r"(dst[27]), + "=r"(dst[28]), "=r"(dst[29]), "=r"(dst[30]), "=r"(dst[31]) + : "r"(src_addr)); +} + +// 32 data path lanes, 32-bit pattern, repeated 16 times +// 32 data path lanes, 32-bit pattern, repeated 8 times +inline __device__ void ldtm_32dp32bit_x8(r32 dst[8], uint32_t src_addr) { + asm volatile("tcgen05.ld.sync.aligned.32x32b.x8.b32" + "{%0, %1, %2, %3," + "%4, %5, %6, %7}," + "[%8];\n" + : "=r"(dst[0]), "=r"(dst[1]), "=r"(dst[2]), "=r"(dst[3]), + "=r"(dst[4]), "=r"(dst[5]), "=r"(dst[6]), "=r"(dst[7]) + : "r"(src_addr)); +} + +// 32 data path lanes, 32-bit pattern, repeated 4 times +// 32 data path lanes, 32-bit pattern, repeated 2 times +inline __device__ void ldtm_32dp32bit_x2(r32 dst[2], uint32_t src_addr) { + asm volatile("tcgen05.ld.sync.aligned.32x32b.x2.b32" + "{%0, %1}," + "[%2];\n" + : "=r"(dst[0]), "=r"(dst[1]) + : "r"(src_addr)); +} + +// 32 data path lanes, 32-bit pattern, repeated 1 times +inline __device__ void ldtm_32dp32bit_x1(r32 dst[1], uint32_t src_addr) { + asm volatile("tcgen05.ld.sync.aligned.32x32b.x1.b32" + "{%0}," + "[%1];\n" + : "=r"(dst[0]) + : "r"(src_addr)); +} + +// CUtensorMap, when issued using bulk copy async functions, are cached in +// constant cache. But the producer kernel may have directly written to global +// memory, without invalidating this constant cache. This acquire fence, +// invalidates the memory address in constant cache, using UTMACCTL.IV. +inline __device__ void tma_descriptor_fence_acquire(CUtensorMap const *p_desc) { +#if (__CUDA_ARCH__ >= 900) && (CUDACC_VERSION >= 123) + uint64_t gmem_int_desc = reinterpret_cast(p_desc); + asm volatile("fence.proxy.tensormap::generic.acquire.gpu [%0], 128;\n" + : + : "l"(gmem_int_desc) + : "memory"); + asm volatile("cvta.global.u64 %0, %0;\n" + : + : "l"(gmem_int_desc), "l"(gmem_int_desc) + : "memory"); +#endif +} + +inline __device__ void +utmaldg_4d_tiled(const void *p_desc, + uint32_t urb0, // smem offset + uint32_t urb1, // smem barrier offset + int32_t urb2, // m + int32_t urb3, // n + int32_t urb4, // b + int32_t urb5, // extra coord + const uint32_t elect_one, + const uint64_t mem_desc = MEM_DESC_DEFAULT) { + if (elect_one) { +#if __CUDA_ARCH__ >= 1200 && __CUDA_ARCH__ < 1300 + asm volatile( + "cp.async.bulk.tensor.4d.shared::cta.global.mbarrier::complete_tx::" + "bytes.L2::cache_hint [%0], [%1, {%2, %3, %4, %5}], [%6], %7;\n" ::"r"( + urb0), + "l"(reinterpret_cast(p_desc)), "r"(urb2), "r"(urb3), + "r"(urb4), "r"(urb5), "r"(urb1), "l"(mem_desc) + : "memory"); +#else + asm volatile( + "cp.async.bulk.tensor.4d.shared::cluster.global.mbarrier::complete_tx::" + "bytes.L2::cache_hint [%0], [%1, {%2, %3, %4, %5}], [%6], %7;\n" ::"r"( + urb0), + "l"(reinterpret_cast(p_desc)), "r"(urb2), "r"(urb3), + "r"(urb4), "r"(urb5), "r"(urb1), "l"(mem_desc) + : "memory"); +#endif + } +} + +inline __device__ void +utmaldg_4d_tiled_multicast(const void *p_desc, + uint32_t urb0, // smem offset + uint32_t urb1, // smem barrier offset + int32_t urb2, // m + int32_t urb3, // n + int32_t urb4, // b + int32_t urb5, // extra coord + const uint16_t mcast_mask, const uint32_t elect_one, + const uint64_t mem_desc = MEM_DESC_DEFAULT) { + if (elect_one) { + asm volatile("cp.async.bulk.tensor.4d.shared::cluster.global.mbarrier::" + "complete_tx::bytes.multicast::cluster.L2::cache_hint [%0], " + "[%1, {%2, %3, %4, %5}], [%6], %7, %8;\n" ::"r"(urb0), + "l"(reinterpret_cast(p_desc)), "r"(urb2), "r"(urb3), + "r"(urb4), "r"(urb5), "r"(urb1), "h"(mcast_mask), "l"(mem_desc) + : "memory"); + } +} + +#if (__CUDA_ARCH__ >= 1000) && (CUDACC_VERSION >= 128) +inline __device__ void +utmaldg_4d_tiled_2cta(const void *p_desc, + uint32_t urb0, // smem offset + uint32_t urb1, // smem barrier offset + int32_t urb2, // m + int32_t urb3, // n + int32_t urb4, // b + int32_t urb5, // extra coord + const uint32_t elect_one, + const uint64_t mem_desc = MEM_DESC_DEFAULT) { + if (elect_one) { + asm volatile("cp.async.bulk.tensor.4d.cta_group::2.shared::cluster.global." + "mbarrier::complete_tx::bytes.L2::cache_hint [%0], [%1, {%2, " + "%3, %4, %5}], [%6], %7;\n" ::"r"(urb0), + "l"(reinterpret_cast(p_desc)), "r"(urb2), "r"(urb3), + "r"(urb4), "r"(urb5), "r"(urb1 & MMA_PEER_BIT_MASK), + "l"(mem_desc) + : "memory"); + } +} + +inline __device__ void +utmaldg_4d_tiled_multicast_2cta(const void *p_desc, + uint32_t urb0, // smem offset + uint32_t urb1, // smem barrier offset + int32_t urb2, // m + int32_t urb3, // n + int32_t urb4, // b + int32_t urb5, // extra coord + const uint16_t mcast_mask, + const uint32_t elect_one, + const uint64_t mem_desc = MEM_DESC_DEFAULT) { + if (elect_one) { + asm volatile( + "cp.async.bulk.tensor.4d.cta_group::2.shared::cluster.global.mbarrier::" + "complete_tx::bytes.multicast::cluster.L2::cache_hint [%0], [%1, {%2, " + "%3, %4, %5}], [%6], %7, %8;\n" ::"r"(urb0), + "l"(reinterpret_cast(p_desc)), "r"(urb2), "r"(urb3), + "r"(urb4), "r"(urb5), "r"(urb1 & MMA_PEER_BIT_MASK), "h"(mcast_mask), + "l"(mem_desc) + : "memory"); + } +} +#endif + +inline __device__ void +utmastg_atomicAdd_4d_tiled(const void *p_desc, uint32_t smem_offset, + int32_t urb0, // m + int32_t urb1, // n + int32_t urb2, // h + int32_t urb3, // b + const uint32_t elect_one, + uint64_t mem_desc = MEM_DESC_DEFAULT) { + if (elect_one) { + asm volatile( + "cp.reduce.async.bulk.tensor.4d.global.shared::cta.add.bulk_group [%0, " + "{%1, %2, %3, %4}], [%5];\n" ::"l"(reinterpret_cast(p_desc)), + "r"(urb0), "r"(urb1), "r"(urb2), "r"(urb3), "r"(smem_offset) + : "memory"); + } +} + +inline __device__ void utmastg_4d_tiled(const void *p_desc, + uint32_t smem_offset, + int32_t urb0, // m + int32_t urb1, // n + int32_t urb2, // h + int32_t urb3, // b + uint64_t mem_desc = MEM_DESC_DEFAULT) { + asm volatile( + "cp.async.bulk.tensor.4d.global.shared::cta.bulk_group [%0, {%1, %2, %3, " + "%4}], [%5];\n" ::"l"(reinterpret_cast(p_desc)), + "r"(urb0), "r"(urb1), "r"(urb2), "r"(urb3), "r"(smem_offset) + : "memory"); +} + +inline __device__ void utmastg_5d_tiled(const void *p_desc, + uint32_t smem_offset, + int32_t urb0, // m + int32_t urb1, // n + int32_t urb2, // h + int32_t urb3, // b + int32_t urb4, // lean_tile_id + uint64_t mem_desc = MEM_DESC_DEFAULT) { + asm volatile( + "cp.async.bulk.tensor.5d.global.shared::cta.bulk_group [%0, {%1, %2, %3, " + "%4, %5}], [%6];\n" ::"l"(reinterpret_cast(p_desc)), + "r"(urb0), "r"(urb1), "r"(urb2), "r"(urb3), "r"(urb4), "r"(smem_offset) + : "memory"); +} + +// mma_loop_op 2 includes + +// global_load_shared_store_op 4 includes + +// shared_load_op 5 includes + +// shared_load_op 6 includes + +// global_load_shared_store_op 7 includes + +// mma_op 8 includes + +// global_load_op 10 includes + +// pointwise_calc_op 9 includes + +// softmax_op 12 includes + +// global_store_op 13 includes + +// global_store_op 14 includes + +// shared_load_op 15 includes + +// global_load_shared_store_op 16 includes + +// mma_op 17 includes + +// global_store_op 20 includes + +// output_loop_op 3 includes + +// shared_store_op 18 includes + +// shared_load_op 19 includes +} // namespace fort +)KERNEL" + R"KERNEL( + +// receive_op 0 types +using namespace fort; + +static constexpr int THREADS_PER_WARP_0 = 32; +static constexpr int WARPS_PER_GROUP_0 = FORT_MIN(4, 512 / THREADS_PER_WARP_0); +static constexpr int THREADS_PER_WARP_GROUP_0 = + THREADS_PER_WARP_0 * WARPS_PER_GROUP_0; +static constexpr int BITS_PER_REGISTER_0 = 32; // NOTE: sizeof(uint) +static constexpr int BYTES_PER_REGISTER_0 = BITS_PER_REGISTER_0 / 8; +static constexpr int THREADS_PER_GROUP_0 = + WARPS_PER_GROUP_0 * THREADS_PER_WARP_0; + +// mma_pipeline_op 1 types +static constexpr int BYTES_PER_BANK_1 = 16; +static constexpr int BITS_PER_ELEMENT_1 = 16; +static constexpr int ELEMS_PER_BANK_1 = + BYTES_PER_BANK_1 * 8 / BITS_PER_ELEMENT_1; +static constexpr int BYTES_PER_ELEMENT_1 = BITS_PER_ELEMENT_1 / 8; +static constexpr int BYTES_PER_FP16_1 = 2; +static constexpr int ELEMS_PER_VECTOR_1 = + BYTES_PER_REGISTER_0 / BYTES_PER_ELEMENT_1; +static constexpr int BITS_PER_ACC_1 = 32; +static constexpr int BYTES_PER_ACC_1 = BITS_PER_ACC_1 / 8; +static constexpr int TILE_M_1 = 128; +static constexpr int TILE_N_1 = 128; +static constexpr int TILE_K_1 = 64; +static constexpr int TILE_O_1 = FORT_MIN(TILE_K_1, 128); +static constexpr int TILE_V_1 = TILE_K_1; +static constexpr int TILE_DV_1 = FORT_MIN(TILE_K_1, 128); +static constexpr int WARP_TILE_M_1 = 16; +static constexpr int WARP_TILE_N_1 = 16; +static constexpr int WARP_TILE_K_1 = 32 / BYTES_PER_ELEMENT_1; +static constexpr int WARPS_M_1 = 64 / 16; +static constexpr int WARPS_N_1 = 16 / 16; +static constexpr int WARPS_K_1 = 16 / 16; +static constexpr int STAGES_1 = 1; +static constexpr int GROUPS_M_1 = 1; +static constexpr int GMMA_TILE_M_1 = WARPS_M_1 * WARP_TILE_M_1; +static constexpr int GMMA_TILE_K_1 = WARPS_K_1 * WARP_TILE_K_1; +static constexpr int WARP_TILES_M_1 = TILE_M_1 / WARP_TILE_M_1; +static constexpr int WARP_TILES_N_1 = TILE_N_1 / WARP_TILE_N_1; +static constexpr int WARP_TILES_K_1 = TILE_K_1 / WARP_TILE_K_1; +static constexpr int WARP_REGS_1 = TILE_N_1; +static constexpr int REGS_M_1 = WARP_TILES_M_1 / WARPS_M_1; +static constexpr int REGS_N_1 = WARP_TILES_N_1 / WARPS_N_1; +static constexpr int REGS_K_1 = WARP_TILES_K_1 / WARPS_K_1; +static constexpr int REGS_K_FOR_DP_1 = TILE_V_1 / 16; +static constexpr int REGS_dQ_1 = TILE_O_1 / 16; +static constexpr int REGS_O_1 = REGS_K_1 * 1; + +static constexpr int CGA_M_1 = 1; +static constexpr int CGA_N_1 = 1; +static constexpr int CTA_MMA_1 = 1; +static constexpr int M_TILES_PER_OUTPUT_TILE_1 = 4; +static constexpr int N_TILES_PER_OUTPUT_TILE_1 = 1; + +static constexpr int CTA_TILE_M_1 = TILE_M_1; +static constexpr int CTA_TILE_N_1 = 64; +static constexpr int CTA_TILE_K_1 = 128; + +static constexpr float ln2 = 0.6931471805599453094f; +static constexpr float inv_ln2 = 1.4426950408889634074f; + +static constexpr uint32_t SchedulerPipelineStageCount = 1; +static constexpr uint32_t AccumulatorPipelineStageCount = 1; +static constexpr uint32_t SmemPipelineStageCount = 1; +static constexpr uint32_t num_columns_per_tmem = 512; +static constexpr uint32_t columns_per_allocation_slice = 32; +static constexpr int stages_q = 2; +static constexpr int stages_kv = 2 * CTA_MMA_1; +static constexpr int Tiles_Q = 2; + +static constexpr int SoftmaxWarpGroups = 2; +static constexpr int CorrectionWarpGroups = 1; +static constexpr int SoftmaxWarps = SoftmaxWarpGroups * 4; +static constexpr int CorrectionWarps = 4; + +static constexpr int MmaWarpRegs = 40; // warp 12 +static constexpr int TmaldgWarpRegs = 40; // warp 13 +static constexpr int TmastgWarpRegs = 40; // warp 14 +static constexpr int SchedulerWarpRegs = 40; // warp 15 +static constexpr int CorrectionWarpRegs = 88; // warps 8-11 +static constexpr int SoftmaxWarpRegs = 192; // warps 0-7 + +static constexpr int threads_per_warp = 32; +static constexpr int warps_per_group = 4; +static constexpr int threads_per_group = threads_per_warp * warps_per_group; +static constexpr int threads_per_cta = + threads_per_warp * (warps_per_group + SoftmaxWarps + CorrectionWarps); + +static constexpr int BUFFERS_Q = stages_q; +static constexpr int BUFFERS_K = stages_kv; +static constexpr int BUFFERS_V = stages_kv; +static constexpr int BUFFERS_S = stages_kv; + +using ElementQ = fort::bfloat16_t; +using ElementK = fort::bfloat16_t; +using ElementV = fort::bfloat16_t; +using ElementS = fort::bfloat16_t; +using ElementO = fort::bfloat16_t; +using ElementAccumulator = float; + +static constexpr int BYTES_PER_ELEMENT = 2; +static constexpr int BYTES_PER_ACC = 4; + +static constexpr int BMM1_TILE_M = TILE_M_1; +static constexpr int BMM1_TILE_N = 128; +static constexpr int BMM1_TILE_K = 64; + +static constexpr int BMM2_TILE_M = TILE_M_1; +static constexpr int BMM2_TILE_N = 64; +static constexpr int BMM2_TILE_K = 128; + +static constexpr uint32_t qBufferElems = BMM1_TILE_M * BMM1_TILE_K; +static constexpr uint32_t kBufferElems = BMM1_TILE_N * BMM1_TILE_K / CTA_MMA_1; +static constexpr uint32_t vBufferElems = BMM2_TILE_N * BMM2_TILE_K / CTA_MMA_1; +static constexpr uint32_t oBufferElems = + FORT_MAX(8, BMM2_TILE_M) * (BMM2_TILE_N + 0); +static constexpr uint32_t sBufferElems = FORT_MAX(8, BMM1_TILE_M) * BMM1_TILE_N; + +static constexpr uint32_t qTmaTransactionBytes = + qBufferElems * sizeof(ElementQ) * CTA_MMA_1; +static constexpr uint32_t kTmaTransactionBytes = + kBufferElems * sizeof(ElementK) * CTA_MMA_1; +static constexpr uint32_t vTmaTransactionBytes = + vBufferElems * sizeof(ElementV) * CTA_MMA_1; + +static constexpr int UTCMMA_TILE_M = 128; +static constexpr int UTCMMA_TILE_N = 128; +static constexpr int UTCMMA_TILE_K = 32 / BYTES_PER_ELEMENT; + +static constexpr int BMM1_XMMAS_K = BMM1_TILE_K / UTCMMA_TILE_K; +static constexpr int BMM2_XMMAS_K = BMM2_TILE_K / UTCMMA_TILE_K; +static constexpr int BYTES_PER_MMA_K = UTCMMA_TILE_K * BYTES_PER_ELEMENT; + +static constexpr int SMEM_BUFFER_SIZE_Q = + BMM1_TILE_M * BMM1_TILE_K * BYTES_PER_ELEMENT; +static constexpr int SMEM_BUFFER_SIZE_K = + BMM1_TILE_N * BMM1_TILE_K * BYTES_PER_ELEMENT / CTA_MMA_1; +static constexpr int SMEM_BUFFER_SIZE_V = + BMM2_TILE_N * BMM2_TILE_K * BYTES_PER_ELEMENT / CTA_MMA_1; +static constexpr int SMEM_BUFFER_SIZE_S = + FORT_MAX(8, BMM1_TILE_M) * BMM1_TILE_N * BYTES_PER_ELEMENT; + +static constexpr int SOFTMAX_BARRIER = 1; +static constexpr int BAND_BIAS_BARRIER = 5; +static constexpr int TMEM_ALLOC_SOFTMAX_BARRIER = 6; +static constexpr int TMEM_ALLOC_CORRECTION_BARRIER = 7; + +struct SharedStorage { + alignas(1024) ElementQ smem_Q[qBufferElems * stages_q]; + alignas(1024) ElementK smem_K[kBufferElems * stages_kv]; + alignas(1024) ElementV smem_V[vBufferElems * stages_kv]; + alignas(1024) ElementO smem_O[oBufferElems]; + + alignas(16) uint64_t tma_o_0_empty_mbar[1]; + + alignas(16) uint64_t tma_k_empty_mbar[stages_kv]; + alignas(16) uint64_t tma_k_full_mbar[stages_kv]; + + alignas(16) uint64_t tma_v_empty_mbar[stages_kv]; + alignas(16) uint64_t tma_v_full_mbar[stages_kv]; + + alignas(16) uint64_t tma_q_empty_mbar[stages_q]; + alignas(16) uint64_t tma_q_full_mbar[stages_q]; + + alignas(16) uint64_t bmm2_tile0_done_mbar[1]; + alignas(16) uint64_t bmm2_tile0_ready_mbar[2]; + + alignas(16) uint64_t stat_tile0_full_mbar[1]; + alignas(16) uint64_t stat_tile0_empty_mbar[1]; + alignas(16) uint64_t final_stat_tile0_empty_mbar[1]; + alignas(16) uint64_t + tma_o_0_full_mbar[FORT_DIV_UP(TILE_O_1 * sizeof(ElementO), 128)]; + alignas(16) uint64_t + tma_o_1_full_mbar[FORT_DIV_UP(TILE_O_1 * sizeof(ElementO), 128)]; + + alignas(16) uint64_t epilogue_done_mbar[1]; + alignas(16) uint64_t tmaldg_tile_started_mbar[1]; + + alignas(16) uint64_t bmm1_tile0_done_mbar[1]; + alignas(16) uint64_t bmm1_tile1_done_mbar[1]; + + alignas(16) uint64_t bmm2_tile1_done_mbar[1]; + alignas(16) uint64_t bmm2_tile1_ready_mbar[2]; + + alignas(16) uint64_t stat_tile1_full_mbar[1]; + alignas(16) uint64_t stat_tile1_empty_mbar[1]; + alignas(16) uint64_t final_stat_tile1_empty_mbar[1]; + + alignas(16) uint64_t empty_mainloop_mbar[1]; + + alignas(16) uint64_t scheduler_mbar[SchedulerPipelineStageCount]; + alignas(16) uint64_t read_tile_id_done_mbar[SchedulerPipelineStageCount]; + alignas(16) uint64_t tmem_dealloc_smem_bar[1]; + + uint32_t tmem_base_ptrs; + + alignas(16) uint32_t tile_id[8 * SchedulerPipelineStageCount]; +}; + +// global_load_shared_store_op 4 types +static constexpr int BITS_PER_ELEMENT_4 = 16; +static constexpr int BYTES_PER_ELEMENT_4 = BITS_PER_ELEMENT_4 / 8; +static constexpr int BYTES_PER_ACCESS_4 = 16; +static constexpr int TILE_M_4 = 128; +static constexpr int TILE_N_4 = 64; +static constexpr int BYTES_PER_SMEM_4 = + TILE_M_4 * TILE_N_4 * BYTES_PER_ELEMENT_4; + +// shared_load_op 5 types +static constexpr int BITS_PER_ELEMENT_5 = 16; +static constexpr int BYTES_PER_ELEMENT_5 = 2; +static constexpr int BYTES_PER_QUAD_5 = 16; +static constexpr int WARP_TILE_M_5 = 16; +static constexpr int WARP_TILE_N_5 = 16; +static constexpr int TILE_M_5 = 128; +static constexpr int TILE_N_5 = 64; +static constexpr int BYTES_PER_SMEM_5 = + TILE_M_5 * TILE_N_5 * BYTES_PER_ELEMENT_5; +static constexpr int WARPS_M_5 = 4; +static constexpr int WARPS_N_5 = 1; +static constexpr int WARP_TILES_M_5 = TILE_M_5 / WARP_TILE_M_5; +static constexpr int WARP_TILES_N_5 = TILE_N_5 / WARP_TILE_N_5; +static constexpr int THREADS_PER_WARP_TILE_M_5 = 8; +static constexpr int THREADS_PER_WARP_TILE_N_5 = 4; +static constexpr int REGS_M_5 = WARP_TILES_M_5 / WARPS_M_5; +static constexpr int REGS_N_5 = WARP_TILES_N_5 / WARPS_N_5; +static constexpr int BYTES_PER_LD_5 = 128; +static constexpr int SWIZZLE_SCALE_5 = + FORT_MIN(BYTES_PER_LD_5 / 16, WARPS_PER_GROUP_0 * 4); + +// shared_load_op 6 types +static constexpr int BITS_PER_ELEMENT_6 = 16; +static constexpr int BYTES_PER_ELEMENT_6 = 2; +static constexpr int BYTES_PER_QUAD_6 = 16; +static constexpr int WARP_TILE_M_6 = 16; +static constexpr int WARP_TILE_N_6 = 16; +static constexpr int TILE_M_6 = 128; +static constexpr int TILE_N_6 = 64; +static constexpr int BYTES_PER_SMEM_6 = + TILE_M_6 * TILE_N_6 * BYTES_PER_ELEMENT_6; +static constexpr int WARPS_M_6 = 1; +static constexpr int WARPS_N_6 = 1; +static constexpr int WARP_TILES_M_6 = TILE_M_6 / WARP_TILE_M_6; +static constexpr int WARP_TILES_N_6 = TILE_N_6 / WARP_TILE_N_6; +static constexpr int THREADS_PER_WARP_TILE_M_6 = 8; +static constexpr int THREADS_PER_WARP_TILE_N_6 = 4; +static constexpr int REGS_M_6 = WARP_TILES_M_6 / WARPS_M_6; +static constexpr int REGS_N_6 = WARP_TILES_N_6 / WARPS_N_6; +static constexpr int BYTES_PER_LD_6 = 128; +static constexpr int SWIZZLE_SCALE_6 = + FORT_MIN(BYTES_PER_LD_6 / 16, WARPS_PER_GROUP_0 * 4); + +// global_load_shared_store_op 7 types +static constexpr int BITS_PER_ELEMENT_7 = 16; +static constexpr int BYTES_PER_ELEMENT_7 = BITS_PER_ELEMENT_7 / 8; +static constexpr int BYTES_PER_ACCESS_7 = 16; +static constexpr int TILE_M_7 = 128; +static constexpr int TILE_N_7 = 64; +static constexpr int BYTES_PER_SMEM_7 = + TILE_M_7 * TILE_N_7 * BYTES_PER_ELEMENT_7; + +// mma_op 8 types +static constexpr int ROWS_PER_CORE_MATRIX_A_8 = 8; +static constexpr int COLS_PER_CORE_MATRIX_A_8 = 8; +static constexpr int ROWS_PER_CORE_MATRIX_B_8 = 8; +static constexpr int COLS_PER_CORE_MATRIX_B_8 = 8; + +// global_load_op 10 types +static constexpr int BITS_PER_ELEMENT_10 = 32; +static constexpr int BYTES_PER_ELEMENT_10 = 4; +static constexpr int REGISTERS_PER_VECTOR_10 = 1; +static constexpr int REGISTERS_PER_ACCESS_10 = 1; + +// pointwise_calc_op 9 types +static constexpr int WARP_TILE_M_9 = 16; +static constexpr int WARP_TILE_N_9 = 16; +static constexpr int TILE_M_9 = 128; +static constexpr int TILE_N_9 = 128; +static constexpr int WARPS_M_9 = 4; +static constexpr int WARPS_N_9 = 1; +static constexpr int WARP_TILES_M_9 = TILE_M_9 / WARP_TILE_M_9; +static constexpr int WARP_TILES_N_9 = TILE_N_9 / WARP_TILE_N_9; +static constexpr int WARP_REGS_9 = WARP_REGS_1; +static constexpr int REGS_M_9 = WARP_TILES_M_9 / WARPS_M_9; +static constexpr int REGS_N_9 = WARP_TILES_N_9 / WARPS_N_9; + +// mha_mask_op 11 types +static constexpr int WARP_TILE_M_11 = 16; +static constexpr int WARP_TILE_N_11 = 16; +static constexpr int TILE_M_11 = 128; +static constexpr int TILE_N_11 = 128; +static constexpr int WARPS_M_11 = 4; +static constexpr int WARPS_N_11 = 1; +static constexpr int WARP_TILES_M_11 = TILE_M_11 / WARP_TILE_M_11; +static constexpr int WARP_TILES_N_11 = TILE_N_11 / WARP_TILE_N_11; +static constexpr int WARP_REGS_11 = 8; +static constexpr int REGS_M_11 = WARP_TILES_M_11 / WARPS_M_11; +static constexpr int REGS_N_11 = WARP_TILES_N_11 / WARPS_N_11; +inline __device__ bool compute_diagonal_band_mask_11( +)KERNEL" + R"KERNEL( + const int row, const int col, const int actual_seqlen_kv, + const int actual_seqlen_q, const int shift_right_bound, + const int left_bound) { + constexpr bool is_bottom_right_alignment = false; + constexpr bool is_shift_right_bound = false; + constexpr bool is_right_bound = true; + constexpr bool is_left_bound = false; + constexpr bool need_oob_check = false; + + const int diag = is_bottom_right_alignment + ? row + (actual_seqlen_kv - actual_seqlen_q) + : row; + const int shifted_right_bound = + is_shift_right_bound ? diag + shift_right_bound : diag; + const bool right_bound_mask = + is_right_bound ? col <= shifted_right_bound : true; + const bool left_bound_mask = is_left_bound ? col + left_bound > diag : true; + const bool oob_check = + need_oob_check ? (row < actual_seqlen_q && col < actual_seqlen_kv) : true; + return right_bound_mask && left_bound_mask && oob_check; +} + +// softmax_op 12 types +static constexpr int BYTES_PER_ELEMENT_12 = 2; +static constexpr int WARP_TILE_M_12 = 16; +static constexpr int WARP_TILE_N_12 = 16; +static constexpr int TILE_M_12 = 128; +static constexpr int TILE_N_12 = 128; +static constexpr int WARPS_M_12 = 4; +static constexpr int WARPS_N_12 = 1; +static constexpr int WARP_TILES_M_12 = TILE_M_12 / WARP_TILE_M_12; +static constexpr int WARP_TILES_N_12 = TILE_N_12 / WARP_TILE_N_12; +static constexpr int THREADS_PER_WARP_TILE_M_12 = 8; +static constexpr int THREADS_PER_WARP_TILE_N_12 = 4; +static constexpr int WARP_REGS_12 = 8; +static constexpr int REGS_M_12 = WARP_TILES_M_12 / WARPS_M_12; +static constexpr int REGS_N_12 = WARP_TILES_N_12 / WARPS_N_12; +static constexpr int ROWS_PER_THREAD_12 = + TILE_M_12 / WARPS_M_12 / THREADS_PER_WARP_TILE_M_12; + +// global_store_op 13 types +static constexpr int BYTES_PER_ELEMENT_13 = 4; +static constexpr int WARP_TILE_M_13 = 16; +static constexpr int WARP_TILE_N_13 = 16; +static constexpr int TILE_M_13 = 128; +static constexpr int TILE_N_13 = 128; +static constexpr int WARPS_M_13 = 4; +static constexpr int WARPS_N_13 = 1; +static constexpr int WARP_TILES_M_13 = TILE_M_13 / WARP_TILE_M_13; +static constexpr int WARP_TILES_N_13 = TILE_N_13 / WARP_TILE_N_13; +static constexpr int THREADS_PER_WARP_TILE_M_13 = 8; +static constexpr int THREADS_PER_WARP_TILE_N_13 = 4; +static constexpr int ROWS_PER_THREAD_13 = + TILE_M_13 / WARPS_M_13 / THREADS_PER_WARP_TILE_M_13; + +// global_store_op 14 types +static constexpr int BYTES_PER_ELEMENT_14 = 4; +static constexpr int WARP_TILE_M_14 = 16; +static constexpr int WARP_TILE_N_14 = 16; +static constexpr int TILE_M_14 = 128; +static constexpr int TILE_N_14 = 128; +static constexpr int WARPS_M_14 = 4; +static constexpr int WARPS_N_14 = 1; +static constexpr int WARP_TILES_M_14 = TILE_M_14 / WARP_TILE_M_14; +static constexpr int WARP_TILES_N_14 = TILE_N_14 / WARP_TILE_N_14; +static constexpr int THREADS_PER_WARP_TILE_M_14 = 8; +static constexpr int THREADS_PER_WARP_TILE_N_14 = 4; +static constexpr int ROWS_PER_THREAD_14 = + TILE_M_14 / WARPS_M_14 / THREADS_PER_WARP_TILE_M_14; + +// shared_load_op 15 types +static constexpr int BITS_PER_ELEMENT_15 = 16; +static constexpr int BYTES_PER_ELEMENT_15 = 2; +static constexpr int BYTES_PER_QUAD_15 = 16; +static constexpr int WARP_TILE_M_15 = 16; +static constexpr int WARP_TILE_N_15 = 16; +static constexpr int TILE_M_15 = 128; +static constexpr int TILE_N_15 = 64; +static constexpr int BYTES_PER_SMEM_15 = + TILE_M_15 * TILE_N_15 * BYTES_PER_ELEMENT_15; +static constexpr int WARPS_M_15 = 1; +static constexpr int WARPS_N_15 = 1; +static constexpr int WARP_TILES_M_15 = TILE_M_15 / WARP_TILE_M_15; +static constexpr int WARP_TILES_N_15 = TILE_N_15 / WARP_TILE_N_15; +static constexpr int THREADS_PER_WARP_TILE_M_15 = 8; +static constexpr int THREADS_PER_WARP_TILE_N_15 = 4; +static constexpr int REGS_M_15 = WARP_TILES_M_15 / WARPS_M_15; +static constexpr int REGS_N_15 = WARP_TILES_N_15 / WARPS_N_15; +static constexpr int BYTES_PER_LD_15 = 128; +static constexpr int SWIZZLE_SCALE_15 = + FORT_MIN(BYTES_PER_LD_15 / 16, WARPS_PER_GROUP_0 * 4); + +// global_load_shared_store_op 16 types +static constexpr int BITS_PER_ELEMENT_16 = 16; +static constexpr int BYTES_PER_ELEMENT_16 = BITS_PER_ELEMENT_16 / 8; +static constexpr int BYTES_PER_ACCESS_16 = 16; +static constexpr int TILE_M_16 = 128; +static constexpr int TILE_N_16 = 64; +static constexpr int BYTES_PER_SMEM_16 = + TILE_M_16 * TILE_N_16 * BYTES_PER_ELEMENT_16; + +// mma_op 17 types +static constexpr int ROWS_PER_CORE_MATRIX_A_17 = 8; +static constexpr int COLS_PER_CORE_MATRIX_A_17 = 8; +static constexpr int ROWS_PER_CORE_MATRIX_B_17 = 8; +static constexpr int COLS_PER_CORE_MATRIX_B_17 = 8; + +// global_store_op 20 types + +// mma_loop_op 2 types +inline __device__ int2 compute_kv_loop_bounds( + const int row_coord, const int ROW_TILE_SIZE, const int COL_TILE_SIZE, + const int actual_seqlen_kv, const int actual_seqlen_q, + const int shift_right_bound, const int left_bound) { + constexpr bool is_right_bound = true; + constexpr bool is_right_bound_bottom_right_alignment = false; + constexpr bool is_shift_right_bound = false; + constexpr bool is_left_bound = false; + constexpr bool is_left_bound_bottom_right_alignment = false; + + const int right_bound_diagonal = + is_right_bound_bottom_right_alignment + ? row_coord + (actual_seqlen_kv - actual_seqlen_q) + : row_coord; + const int shifted_right_bound_diagonal = + is_shift_right_bound ? right_bound_diagonal + shift_right_bound + : right_bound_diagonal; + const int left_bound_diagonal = + is_left_bound_bottom_right_alignment + ? row_coord + (actual_seqlen_kv - actual_seqlen_q) + : row_coord; + + const int kv_loop_left_bound = + is_left_bound + ? FORT_MAX(0, (left_bound_diagonal - left_bound) / COL_TILE_SIZE) + : 0; + const int kv_loop_right_bound = + is_right_bound + ? FORT_MIN(FORT_DIV_UP(shifted_right_bound_diagonal + ROW_TILE_SIZE, + COL_TILE_SIZE), + FORT_DIV_UP(actual_seqlen_kv, COL_TILE_SIZE)) + : FORT_DIV_UP(actual_seqlen_kv, COL_TILE_SIZE); + return make_int2(kv_loop_left_bound, kv_loop_right_bound); +} + +// output_loop_op 3 types +static constexpr int BYTES_PER_BANK_3 = 16; +static constexpr int ELEMENTS_PER_VECTOR_3 = + FORT_MIN(64, CTA_TILE_N_1 / N_TILES_PER_OUTPUT_TILE_1); + +// shared_store_op 18 types + +// shared_load_op 19 types +static constexpr int BITS_PER_ELEMENT_19 = 32 * 1; +static constexpr int BYTES_PER_ELEMENT_19 = BITS_PER_ELEMENT_19 / 8; +static constexpr int BITS_PER_VECTOR_19 = + BITS_PER_ELEMENT_19 * ELEMENTS_PER_VECTOR_3; +static constexpr int BYTES_PER_VECTOR_19 = BITS_PER_VECTOR_19 / 8; +static constexpr int REGISTERS_PER_VECTOR_19 = + BITS_PER_VECTOR_19 / BITS_PER_REGISTER_0; + +// receive_op 0 code + +extern "C" __global__ +__launch_bounds__(512, 1) void cudnn_generated_oss_sdpa_sm100_flash_fprop_f16_knob_7_128x128x64_4x1x1_cga1x1x1_kernel0_0( + const AttentionDescriptor_t attnDesc, const FastDivisor_t num_head_divmod, + const FastDivisor_t l2_minor_divmod, const FastDivisor_t l2_major_divmod, + const FastDivisor_t l2_minor_residual_divmod, const int num_hb_quotient, + const int num_block, const int total_blocks, + __grid_constant__ const CUtensorMap tma_tensor_3, + __grid_constant__ const CUtensorMap tma_tensor_2, float tensor_4, + float tensor_13, void *tensor_7, fort::tensor_descriptor desc_7, + void *tensor_6, fort::tensor_descriptor desc_6, + __grid_constant__ const CUtensorMap tma_tensor_1, + __grid_constant__ const CUtensorMap tma_tensor_5) { + asm volatile(".pragma \"global knob ForceLateCommoning=1\";\n" ::: "memory"); + asm volatile(".pragma \"global knob HoistLate=3\";\n" ::: "memory"); + + extern __shared__ char smem_[]; + SharedStorage &shared_storage = *reinterpret_cast(smem_); + + // receive_op 0 decls + + // mma_pipeline_op 1 decls + const int tid = threadIdx.x % threads_per_group; + const int wid = threadIdx.x / 32; + const int tiw = threadIdx.x % 32; + + const uint32_t elect_one = elect_one_sync(); + + if (wid == 0 && elect_one) { + smem_bar_init(cast_smem_ptr_to_uint(&shared_storage.tma_o_0_empty_mbar[0]), + /* 1 tmastg */ 32); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.tmem_dealloc_smem_bar[0]), + /* 4 epilogue */ warps_per_group * 32 * CTA_MMA_1); + +#pragma unroll + for (int q_stage = 0; q_stage < stages_q; q_stage++) { + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.tma_q_empty_mbar[q_stage]), + /* 1 mma */ 1); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.tma_q_full_mbar[q_stage]), + /* 1 tma */ 1); + } + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.bmm2_tile0_ready_mbar[0]), + /* 4 Softmax */ warps_per_group * 32 * 2 * CTA_MMA_1); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.bmm2_tile0_done_mbar[0]), + /* 1 mma */ 1); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.stat_tile0_full_mbar[0]), + /* 4 Softmax */ warps_per_group * 32); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.stat_tile0_empty_mbar[0]), + /* 4 Softmax */ warps_per_group * 32); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.final_stat_tile0_empty_mbar[0]), + /* 4 Correction */ warps_per_group * 32 * CTA_MMA_1); + +#pragma unroll + for (int i = 0; i < TILE_O_1 * sizeof(ElementO); i += 128) { + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.tma_o_0_full_mbar[i / 128]), + /* 1 correction */ warps_per_group * 32); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.tma_o_1_full_mbar[i / 128]), + /* 1 correction */ warps_per_group * 32); + } + smem_bar_init(cast_smem_ptr_to_uint(&shared_storage.epilogue_done_mbar[0]), + /* 1 tmastg */ 32); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.tmaldg_tile_started_mbar[0]), 32); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.bmm2_tile1_ready_mbar[0]), + /* 4 Softmax */ warps_per_group * 32 * 2 * CTA_MMA_1); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.bmm2_tile0_ready_mbar[1]), + /* 4 Softmax */ warps_per_group * 32 * 1 * CTA_MMA_1); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.bmm2_tile1_ready_mbar[1]), + /* 4 Softmax */ warps_per_group * 32 * 1 * CTA_MMA_1); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.bmm1_tile0_done_mbar[0]), + /* 1 mma */ 1); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.bmm1_tile1_done_mbar[0]), + /* 1 mma */ 1); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.bmm2_tile1_done_mbar[0]), + /* 1 mma */ 1); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.stat_tile1_full_mbar[0]), + /* 4 Softmax */ warps_per_group * 32); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.stat_tile1_empty_mbar[0]), + /* 4 Softmax */ warps_per_group * 32); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.final_stat_tile1_empty_mbar[0]), + /* 4 Correction */ warps_per_group * 32 * CTA_MMA_1); + + smem_bar_init(cast_smem_ptr_to_uint(&shared_storage.empty_mainloop_mbar[0]), + /* 4 Softmax */ warps_per_group * 32 * 1 * CTA_MMA_1); + +#pragma unroll + for (int sched_steps = 0; sched_steps < SchedulerPipelineStageCount; + sched_steps++) { + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.scheduler_mbar[sched_steps]), + /* 1 sched */ 1); + smem_bar_init(cast_smem_ptr_to_uint( + &shared_storage.read_tile_id_done_mbar[sched_steps]), + (SoftmaxWarps + CorrectionWarps + 1 + 1) * CGA_M_1 * + CGA_N_1 + + CGA_M_1 * CGA_N_1 / CTA_MMA_1); + } + +#pragma unroll + for (int kv_stages = 0; kv_stages < stages_kv; kv_stages++) { + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.tma_k_empty_mbar[kv_stages]), + /* 1 mma */ CGA_M_1 / CTA_MMA_1 + CGA_N_1 - 1); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.tma_k_full_mbar[kv_stages]), + /* 1 tma/32 ldgsts */ 1); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.tma_v_full_mbar[kv_stages]), + /* 1 tma/32 ldgsts */ 1); + smem_bar_init( + cast_smem_ptr_to_uint(&shared_storage.tma_v_empty_mbar[kv_stages]), + /* 1 mma */ CGA_M_1 / CTA_MMA_1 + CGA_N_1 - 1); + } + } + __syncthreads(); + +)KERNEL" + R"KERNEL( + // mma_pipeline_op 1 code + + // mma_loop_op 2 code + uint32_t tile_id_and_coord[8] = {0}; + + int actual_seqlen_kv_1 = attnDesc.s_kv; + int actual_seqlen_q_1 = attnDesc.s_q; + int q_row_coord = 0; + int kv_loop_right_bound, kv_loop_left_bound; + + const int left_bound = 0; + const int shift_right_bound = 0; + + uint32_t tile_idx = blockIdx.x; + uint32_t bidhb, l2_mod; + uint32_t block = 0, bidhb_residual = 0; + + fastDivMod(l2_major_divmod, tile_idx, bidhb, l2_mod); + if (bidhb < num_hb_quotient) { + { + fastDivMod(l2_minor_divmod, l2_mod, block, bidhb_residual); + } + } else { + { + fastDivMod(l2_minor_residual_divmod, l2_mod, block, bidhb_residual); + } + } + + int bidhb_actual = bidhb * l2_minor_divmod.val + bidhb_residual; + uint32_t batch_idx, head_idx; + fastDivMod(num_head_divmod, bidhb_actual, batch_idx, head_idx); + block = num_block - 1 - block; + tile_id_and_coord[0] = head_idx; + tile_id_and_coord[1] = batch_idx; + tile_id_and_coord[2] = block; + tile_id_and_coord[3] = 1; + + int head_coord_q_offset_1 = 0; + uint32_t head_coord_1, head_coord_k_1, head_coord_v_1; + uint32_t blocked_row_coord = tile_id_and_coord[2]; + uint32_t head_coord_from_grid_1 = tile_id_and_coord[0]; + uint32_t batch_coord_1 = tile_id_and_coord[1]; + uint32_t lean_tile_id = tile_id_and_coord[0]; + + uint32_t is_valid_tile = true; + + head_coord_1 = head_coord_from_grid_1; + head_coord_k_1 = head_coord_1 / attnDesc.q_heads_per_k; + head_coord_v_1 = head_coord_1 / attnDesc.q_heads_per_v; + + q_row_coord = (blocked_row_coord * Tiles_Q * BMM1_TILE_M); + int2 kv_loop_bounds = compute_kv_loop_bounds( + q_row_coord, BMM1_TILE_M * Tiles_Q * CGA_M_1, TILE_N_1, + actual_seqlen_kv_1, actual_seqlen_q_1, shift_right_bound, left_bound); + kv_loop_left_bound = kv_loop_bounds.x; + kv_loop_right_bound = kv_loop_bounds.y; + if (actual_seqlen_q_1 == 0 || q_row_coord >= actual_seqlen_q_1) { + kv_loop_right_bound = kv_loop_left_bound; + } + + PipelineState sched_state(0, 0); // index, state + + // thread_specialize_op 21 code + if (wid >= 0 && wid < SoftmaxWarps) { + // Softmax + reg_alloc(); + + const uint32_t softmax_gid = wid / 4; + const uint32_t softmax_gid_s_offset = softmax_gid * BMM1_TILE_N; + + uint32_t base_tmem_addr; + named_barrier_wait(TMEM_ALLOC_SOFTMAX_BARRIER, + threads_per_warp * (SoftmaxWarps + 1)); + lds_32(&base_tmem_addr, + cast_smem_ptr_to_uint(&shared_storage.tmem_base_ptrs)); + uint32_t tmem_fp32_S = base_tmem_addr + 0 + softmax_gid_s_offset; + uint32_t tmem_fp16_S = + base_tmem_addr + BMM1_TILE_N / 2 + softmax_gid_s_offset; + uint32_t tmem_Stats = base_tmem_addr + 0 + softmax_gid_s_offset; + + r32 reg_8_0[BMM1_TILE_N]; + r32 fp32_stats[2]; + + // Memory barrier states + uint32_t bmm_mbar_state = 0; + uint32_t stat_mbar_state = 0; + uint32_t epilogue_state = 0; + +// Persistent loop over tiles +#pragma unroll 1 + while (is_valid_tile) { + int p_row_1 = q_row_coord + softmax_gid * TILE_M_1 + tid; + + // global_load_op 10 decls + r32 reg_10_0[1]; + reg_10_0[0] = reinterpret_cast(tensor_4); + const int oob_M_10 = 1; + const int oob_N_10 = 1; + + // pointwise_calc_op 9 decls + + // mha_mask_op 11 decls + + // softmax_op 12 decls + r32 reg_12_0[32]; + float2 total_sum = {0.0f, 0.0f}; + float2 local_sum = {0.0f, 0.0f}; + float total_max = NEG_INFINITY; + float actual_total_max = NEG_INFINITY; + static constexpr float inv_ln2_12 = 1.4426950408889634074f; + float total_max_scale_12 = inv_ln2_12; + const float bmm_scale = reinterpret_cast(tensor_4); + total_max_scale_12 *= bmm_scale; + + uint64_t &bmm1_mbar = softmax_gid == 0 + ? shared_storage.bmm1_tile0_done_mbar[0] + : shared_storage.bmm1_tile1_done_mbar[0]; + uint64_t &stat_empty_mbar = softmax_gid == 0 + ? shared_storage.stat_tile0_empty_mbar[0] + : shared_storage.stat_tile1_empty_mbar[0]; + uint64_t *bmm2_mbar = softmax_gid == 0 + ? &(shared_storage.bmm2_tile0_ready_mbar[0]) + : &(shared_storage.bmm2_tile1_ready_mbar[0]); + uint64_t *stat_full_mbar = + softmax_gid == 0 ? &(shared_storage.stat_tile0_full_mbar[0]) + : &(shared_storage.stat_tile1_full_mbar[0]); + + epilogue_state ^= 1; + wait_barrier(cast_smem_ptr_to_uint(&shared_storage.epilogue_done_mbar[0]), + epilogue_state); + stat_mbar_state ^= 1; + wait_barrier(cast_smem_ptr_to_uint(&stat_empty_mbar), stat_mbar_state); + +#pragma unroll 1 + for (int kv_loop = kv_loop_left_bound; + kv_loop < + kv_loop_right_bound - + FORT_DIV_UP(BMM1_TILE_M * CGA_M_1 * Tiles_Q, BMM1_TILE_N); + kv_loop += 1) { + int p_col_1 = kv_loop * TILE_N_1; + { + + wait_barrier(cast_smem_ptr_to_uint(&bmm1_mbar), bmm_mbar_state); + + r32 stat_reg[BMM1_TILE_N / 64]; +#pragma unroll + for (int ldtm_step = 0; ldtm_step < BMM1_TILE_N / 32; ldtm_step++) { + ldtm_32dp32bit_x32(®_8_0[32 * ldtm_step], + tmem_fp32_S + ldtm_step * 32); + } + constexpr bool use_ldtm_stat_max = false; + + // global_load_op 10 code + + // softmax_op 12 code + float current_max = + row_max_reduction_128_elems(reg_8_0) * total_max_scale_12; + + // Update total max + float total_max_tmp = fmaxf(current_max, total_max); + current_max = total_max; + total_max = total_max_tmp; + + float2 bmm_scale_x_ln2_x2 = + make_float2(total_max_scale_12, total_max_scale_12); + float total_max_scaled = + (total_max == NEG_INFINITY) ? 0.0f : total_max; + float2 minus_scaled_max_x2 = + make_float2(-total_max_scaled, -total_max_scaled); + + static constexpr int kMulPipeCount = 14; // must be multiple of 2 + static constexpr int kSubtractPipeCount = 12; // must be multiple of 2 + static constexpr int kFmaPipeCount = 12; // must be multiple of 2 + static constexpr int kConvertPipeCount = 8; // must be multiple of 2 + static constexpr int kReleasePipeCount = 4; // must be multiple of 2 + static constexpr int kDropoutPipeCount = + 6; // must be multiple of 2 and less than or equal + // kConvertPipeCount + static constexpr int kAddPipeCount = + 4; // must be multiple of 2 and less than kDropoutPipeCount + + // To better overlap EX2 MUFU and emulation, we need to have last + // kE2eRes of every kE2eFreq elements in a row to use emulation while + // the rest go to MUFU. Also, we need to avoid using emulation after + // kE2eLimit elements as it would delay the execution of row sum of + // another softmax warpgroup because of resource contention + static constexpr int kE2eFreq = 16; + static constexpr int kE2eRes = 8; + static constexpr int kE2eLimit = BMM1_TILE_N / 16 * 9; + +#pragma unroll + for (int i = 0; i < kFmaPipeCount; i += 2) { + float2 in = make_float2(reinterpret_cast(reg_8_0[i]), + reinterpret_cast(reg_8_0[i + 1])); + float2 out = ffma2(bmm_scale_x_ln2_x2, in, minus_scaled_max_x2); + + reinterpret_cast(reg_8_0[i + 0]) = out.x; + reinterpret_cast(reg_8_0[i + 1]) = out.y; + } + + // Optimized version of alpha = __expf(current_max - total_max); + float exp_input = (total_max == NEG_INFINITY) + ? NEG_INFINITY + : (current_max - total_max); + float alpha = exp2f(exp_input); + + // Send stats to correction warp + reinterpret_cast(fp32_stats[0]) = alpha; + + // STTM + sttm_32dp32bit_x1(tmem_Stats, &fp32_stats[0]); + fence_view_async_tmem_store(); + arrive_barrier(cast_smem_ptr_to_uint(stat_full_mbar)); + +#pragma unroll + for (int i = 0; i < BMM1_TILE_N; i += 2) { + if (i - kConvertPipeCount == 32) { + sttm_32dp32bit_x16(tmem_fp16_S, ®_12_0[0]); + } + if (i - kConvertPipeCount == 64) { + sttm_32dp32bit_x16(tmem_fp16_S + 16, ®_12_0[16]); + fence_view_async_tmem_store(); + arrive_barrier(cast_smem_ptr_to_uint(bmm2_mbar + 0)); + } + + if (i >= kConvertPipeCount) { + fp32x2_to_bf16x2( + ®_12_0[((i - kConvertPipeCount) / 2) % 32], + reinterpret_cast(®_8_0[i - kConvertPipeCount])); + } + + reinterpret_cast(reg_8_0[i + 0]) = + exp2f(reinterpret_cast(reg_8_0[i + 0])); + + + if (i + kFmaPipeCount < BMM1_TILE_N) { + float2 in = make_float2( + reinterpret_cast(reg_8_0[i + kFmaPipeCount + 0]), + reinterpret_cast(reg_8_0[i + kFmaPipeCount + 1])); + float2 out = ffma2(bmm_scale_x_ln2_x2, in, minus_scaled_max_x2); + + reinterpret_cast(reg_8_0[i + kFmaPipeCount + 0]) = out.x; + reinterpret_cast(reg_8_0[i + kFmaPipeCount + 1]) = out.y; + } + + reinterpret_cast(reg_8_0[i + 1]) = + exp2f(reinterpret_cast(reg_8_0[i + 1])); + + if (i == kE2eLimit) { + if (softmax_gid == 1) { + named_barrier_arrive(SOFTMAX_BARRIER, 256); + named_barrier_wait(SOFTMAX_BARRIER + 1, 256); + } + } + } + +#pragma unroll + for (int i = BMM1_TILE_N - kConvertPipeCount; i < BMM1_TILE_N; + i += 2) { + fp32x2_to_bf16x2(®_12_0[(i / 2) % 32], + reinterpret_cast(®_8_0[i])); + } + sttm_32dp32bit_x32(tmem_fp16_S + BMM1_TILE_N / 64 * 16, + ®_12_0[((BMM1_TILE_N / 64) % 2) * 16]); + fence_view_async_tmem_store(); + arrive_barrier( + cast_smem_ptr_to_uint(bmm2_mbar + (BMM1_TILE_N / 64 - 1))); + + stat_mbar_state ^= 1; + if (!smem_bar_peek(cast_smem_ptr_to_uint(&stat_empty_mbar), + stat_mbar_state)) { + wait_barrier(cast_smem_ptr_to_uint(&stat_empty_mbar), + stat_mbar_state); + } + + float2 alpha2 = make_float2(alpha, alpha); + float local_row_sum_0 = 0.0f; + total_sum = fmul2(alpha2, total_sum); + +#pragma unroll + for (int i = 0; i < BMM1_TILE_N; i += 2) { + float2 in = make_float2(reinterpret_cast(reg_8_0[i + 0]), + reinterpret_cast(reg_8_0[i + 1])); + total_sum = fadd2(total_sum, in); + } + if (softmax_gid == 0) { + named_barrier_arrive(SOFTMAX_BARRIER + 1, 256); + named_barrier_wait(SOFTMAX_BARRIER, 256); + } + named_barrier_wait(SOFTMAX_BARRIER + 2 + softmax_gid, 128); + } + bmm_mbar_state ^= 1; + } // End of mainloop +#pragma unroll 1 + for (int kv_loop = FORT_MAX( + kv_loop_right_bound - + FORT_DIV_UP(BMM1_TILE_M * CGA_M_1 * Tiles_Q, BMM1_TILE_N), + kv_loop_left_bound); + kv_loop < kv_loop_right_bound; kv_loop += 1) { + int p_col_1 = kv_loop * TILE_N_1; + { + + wait_barrier(cast_smem_ptr_to_uint(&bmm1_mbar), bmm_mbar_state); + + r32 stat_reg[BMM1_TILE_N / 64]; +#pragma unroll + for (int ldtm_step = 0; ldtm_step < BMM1_TILE_N / 32; ldtm_step++) { + ldtm_32dp32bit_x32(®_8_0[32 * ldtm_step], + tmem_fp32_S + ldtm_step * 32); + } + constexpr bool use_ldtm_stat_max = false; + +)KERNEL" + R"KERNEL( + // global_load_op 10 code + + // mha_mask_op 11 code + + { +#pragma unroll + for (int i = 0; i < BMM1_TILE_N; i++) { + int row = p_row_1; + int col = p_col_1 + i; + bool mask = compute_diagonal_band_mask_11( + row, col, actual_seqlen_kv_1, actual_seqlen_q_1, + shift_right_bound, left_bound); + if (!mask) { + reinterpret_cast(reg_8_0[i]) = tensor_13; + } + } + } + + // softmax_op 12 code + float current_max = + row_max_reduction_128_elems(reg_8_0) * total_max_scale_12; + + // Update total max + float total_max_tmp = fmaxf(current_max, total_max); + current_max = total_max; + total_max = total_max_tmp; + + float2 bmm_scale_x_ln2_x2 = + make_float2(total_max_scale_12, total_max_scale_12); + float total_max_scaled = + (total_max == NEG_INFINITY) ? 0.0f : total_max; + float2 minus_scaled_max_x2 = + make_float2(-total_max_scaled, -total_max_scaled); + + static constexpr int kMulPipeCount = 14; // must be multiple of 2 + static constexpr int kSubtractPipeCount = 12; // must be multiple of 2 + static constexpr int kFmaPipeCount = 12; // must be multiple of 2 + static constexpr int kConvertPipeCount = 8; // must be multiple of 2 + static constexpr int kReleasePipeCount = 4; // must be multiple of 2 + static constexpr int kDropoutPipeCount = + 6; // must be multiple of 2 and less than or equal + // kConvertPipeCount + static constexpr int kAddPipeCount = + 4; // must be multiple of 2 and less than kDropoutPipeCount + + // To better overlap EX2 MUFU and emulation, we need to have last + // kE2eRes of every kE2eFreq elements in a row to use emulation while + // the rest go to MUFU. Also, we need to avoid using emulation after + // kE2eLimit elements as it would delay the execution of row sum of + // another softmax warpgroup because of resource contention + static constexpr int kE2eFreq = 16; + static constexpr int kE2eRes = 8; + static constexpr int kE2eLimit = BMM1_TILE_N / 16 * 9; + +#pragma unroll + for (int i = 0; i < kFmaPipeCount; i += 2) { + float2 in = make_float2(reinterpret_cast(reg_8_0[i]), + reinterpret_cast(reg_8_0[i + 1])); + float2 out = ffma2(bmm_scale_x_ln2_x2, in, minus_scaled_max_x2); + + reinterpret_cast(reg_8_0[i + 0]) = out.x; + reinterpret_cast(reg_8_0[i + 1]) = out.y; + } + + // Optimized version of alpha = __expf(current_max - total_max); + float exp_input = (total_max == NEG_INFINITY) + ? NEG_INFINITY + : (current_max - total_max); + float alpha = exp2f(exp_input); + + // Send stats to correction warp + reinterpret_cast(fp32_stats[0]) = alpha; + + // STTM + sttm_32dp32bit_x1(tmem_Stats, &fp32_stats[0]); + fence_view_async_tmem_store(); + arrive_barrier(cast_smem_ptr_to_uint(stat_full_mbar)); + +#pragma unroll + for (int i = 0; i < BMM1_TILE_N; i += 2) { + if (i - kConvertPipeCount == 32) { + sttm_32dp32bit_x16(tmem_fp16_S, ®_12_0[0]); + } + if (i - kConvertPipeCount == 64) { + sttm_32dp32bit_x16(tmem_fp16_S + 16, ®_12_0[16]); + fence_view_async_tmem_store(); + arrive_barrier(cast_smem_ptr_to_uint(bmm2_mbar + 0)); + } + + if (i >= kConvertPipeCount) { + fp32x2_to_bf16x2( + ®_12_0[((i - kConvertPipeCount) / 2) % 32], + reinterpret_cast(®_8_0[i - kConvertPipeCount])); + } + + reinterpret_cast(reg_8_0[i + 0]) = + exp2f(reinterpret_cast(reg_8_0[i + 0])); + + + if (i + kFmaPipeCount < BMM1_TILE_N) { + float2 in = make_float2( + reinterpret_cast(reg_8_0[i + kFmaPipeCount + 0]), + reinterpret_cast(reg_8_0[i + kFmaPipeCount + 1])); + float2 out = ffma2(bmm_scale_x_ln2_x2, in, minus_scaled_max_x2); + + reinterpret_cast(reg_8_0[i + kFmaPipeCount + 0]) = out.x; + reinterpret_cast(reg_8_0[i + kFmaPipeCount + 1]) = out.y; + } + + reinterpret_cast(reg_8_0[i + 1]) = + exp2f(reinterpret_cast(reg_8_0[i + 1])); + + if (i == kE2eLimit) { + if (softmax_gid == 1) { + named_barrier_arrive(SOFTMAX_BARRIER, 256); + named_barrier_wait(SOFTMAX_BARRIER + 1, 256); + } + } + } + +#pragma unroll + for (int i = BMM1_TILE_N - kConvertPipeCount; i < BMM1_TILE_N; + i += 2) { + fp32x2_to_bf16x2(®_12_0[(i / 2) % 32], + reinterpret_cast(®_8_0[i])); + } + sttm_32dp32bit_x32(tmem_fp16_S + BMM1_TILE_N / 64 * 16, + ®_12_0[((BMM1_TILE_N / 64) % 2) * 16]); + fence_view_async_tmem_store(); + arrive_barrier( + cast_smem_ptr_to_uint(bmm2_mbar + (BMM1_TILE_N / 64 - 1))); + + stat_mbar_state ^= 1; + if (!smem_bar_peek(cast_smem_ptr_to_uint(&stat_empty_mbar), + stat_mbar_state)) { + wait_barrier(cast_smem_ptr_to_uint(&stat_empty_mbar), + stat_mbar_state); + } + + float2 alpha2 = make_float2(alpha, alpha); + float local_row_sum_0 = 0.0f; + total_sum = fmul2(alpha2, total_sum); + +#pragma unroll + for (int i = 0; i < BMM1_TILE_N; i += 2) { + float2 in = make_float2(reinterpret_cast(reg_8_0[i + 0]), + reinterpret_cast(reg_8_0[i + 1])); + total_sum = fadd2(total_sum, in); + } + if (softmax_gid == 0) { + named_barrier_arrive(SOFTMAX_BARRIER + 1, 256); + named_barrier_wait(SOFTMAX_BARRIER, 256); + } + named_barrier_wait(SOFTMAX_BARRIER + 2 + softmax_gid, 128); + } + bmm_mbar_state ^= 1; + } // End of mainloop + + float final_total_sum = total_sum.x + total_sum.y; + + reinterpret_cast(fp32_stats[0]) = total_max; + reinterpret_cast(fp32_stats[1]) = final_total_sum; + // STTM + sttm_32dp32bit_x2(tmem_Stats, &fp32_stats[0]); + fence_view_async_tmem_store(); + arrive_barrier(cast_smem_ptr_to_uint(stat_full_mbar)); + + // This portion of the code may seem strange, please read the + // documentation here: + // Function: work_tile_info_from_workid_response, load_query_response + if (elect_one) { + arrive_barrier(cast_smem_ptr_to_uint( + &(shared_storage.read_tile_id_done_mbar[sched_state.index()]))); + } + // Get next tile id + wait_barrier(cast_smem_ptr_to_uint( + &shared_storage.scheduler_mbar[sched_state.index()]), + sched_state.phase()); + uint32_t tile_id_smem_int_ptr = cast_smem_ptr_to_uint( + &shared_storage.tile_id[sched_state.index() * 8]); + lds_128(tile_id_and_coord, tile_id_smem_int_ptr); + ++sched_state; + + uint32_t tile_idx = tile_id_and_coord[0]; + uint32_t bidhb, l2_mod; + uint32_t block = 0, bidhb_residual = 0; + + fastDivMod(l2_major_divmod, tile_idx, bidhb, l2_mod); + if (bidhb < num_hb_quotient) { + { + fastDivMod(l2_minor_divmod, l2_mod, block, bidhb_residual); + } + } else { + { + fastDivMod(l2_minor_residual_divmod, l2_mod, block, bidhb_residual); + } + } + + int bidhb_actual = bidhb * l2_minor_divmod.val + bidhb_residual; + uint32_t batch_idx, head_idx; + fastDivMod(num_head_divmod, bidhb_actual, batch_idx, head_idx); + block = num_block - 1 - block; + + lean_tile_id = tile_id_and_coord[0]; + + blocked_row_coord = block + 0; + head_coord_from_grid_1 = (head_idx) + 0; + batch_coord_1 = batch_idx; + is_valid_tile = tile_id_and_coord[2] & 1; + + head_coord_1 = head_coord_from_grid_1; + head_coord_k_1 = head_coord_1 / attnDesc.q_heads_per_k; + head_coord_v_1 = head_coord_1 / attnDesc.q_heads_per_v; + + q_row_coord = (blocked_row_coord * Tiles_Q * BMM1_TILE_M); + int2 kv_loop_bounds = compute_kv_loop_bounds( + q_row_coord, BMM1_TILE_M * Tiles_Q * CGA_M_1, TILE_N_1, + actual_seqlen_kv_1, actual_seqlen_q_1, shift_right_bound, left_bound); + kv_loop_left_bound = kv_loop_bounds.x; + kv_loop_right_bound = kv_loop_bounds.y; + if (actual_seqlen_q_1 == 0 || q_row_coord >= actual_seqlen_q_1) { + kv_loop_right_bound = kv_loop_left_bound; + } + } // end of persistent loop + } + + // thread_specialize_op 22 code + else if (wid >= SoftmaxWarps && wid < SoftmaxWarps + CorrectionWarps) { + // Correction + reg_dealloc(); + uint32_t base_tmem_addr; + named_barrier_wait(TMEM_ALLOC_CORRECTION_BARRIER, + threads_per_warp * (CorrectionWarps + 1)); + lds_32(&base_tmem_addr, + cast_smem_ptr_to_uint(&shared_storage.tmem_base_ptrs)); + uint32_t tmem_fp32_O_0 = base_tmem_addr + 256; + uint32_t tmem_fp32_O_1 = base_tmem_addr + 384; + uint32_t tmem_Stats_0 = base_tmem_addr + 0; + uint32_t tmem_Stats_1 = base_tmem_addr + 0 + BMM1_TILE_N; + + static constexpr int local_correction_block_size = 16; + static constexpr int local_epilogue_block_size = + 128 / sizeof(ElementO); // hack as only K=128/64 supported. Both can do + // 128B TMASTG. + +)KERNEL" + R"KERNEL( + float amax_s_1 = 0.f; + float amax_o_1 = 0.f; + + // Registers + uint32_t fp32_stats[2]; + uint32_t fp32_O[local_correction_block_size]; + + // Memory barrier states + uint32_t epilogue_state = 0; + uint32_t bmm_mbar_state = 0; + uint32_t stat_mbar_state = 0; + +// Persistent loop over tiles +#pragma unroll 1 + while (is_valid_tile) { + if (elect_one) { + arrive_barrier(cast_smem_ptr_to_uint( + &(shared_storage.read_tile_id_done_mbar[sched_state.index()]))); + } + + if (kv_loop_left_bound < kv_loop_right_bound) { + // No correction for iter 0 + arrive_barrier( + cast_smem_ptr_to_uint(&(shared_storage.bmm2_tile0_ready_mbar[0]))); + arrive_barrier( + cast_smem_ptr_to_uint(&(shared_storage.bmm2_tile1_ready_mbar[0]))); + + { + wait_barrier( + cast_smem_ptr_to_uint(&shared_storage.stat_tile0_full_mbar[0]), + stat_mbar_state); + arrive_barrier(cast_smem_ptr_to_uint( + &(shared_storage.stat_tile0_empty_mbar[0]))); + } + { + wait_barrier( + cast_smem_ptr_to_uint(&shared_storage.stat_tile1_full_mbar[0]), + stat_mbar_state); + arrive_barrier(cast_smem_ptr_to_uint( + &(shared_storage.stat_tile1_empty_mbar[0]))); + } + stat_mbar_state ^= 1; + } else { + arrive_barrier( + cast_smem_ptr_to_uint(&(shared_storage.empty_mainloop_mbar[0]))); + } + +#pragma unroll 1 + for (int kv_loop = kv_loop_left_bound + 1; kv_loop < kv_loop_right_bound; + kv_loop += 1) { +#pragma unroll + for (int sub_tile_id = 0; sub_tile_id < 2; ++sub_tile_id) { + uint32_t tmem_Stats = sub_tile_id == 0 ? tmem_Stats_0 : tmem_Stats_1; + uint32_t tmem_fp32_O = + sub_tile_id == 0 ? tmem_fp32_O_0 : tmem_fp32_O_1; + + uint64_t &stat_full_mbar = + sub_tile_id == 0 ? shared_storage.stat_tile0_full_mbar[0] + : shared_storage.stat_tile1_full_mbar[0]; + uint64_t &bmm_done_mbar = + sub_tile_id == 0 ? shared_storage.bmm2_tile0_done_mbar[0] + : shared_storage.bmm2_tile1_done_mbar[0]; + uint64_t *stat_empty_mbar = + sub_tile_id == 0 ? &(shared_storage.stat_tile0_empty_mbar[0]) + : &(shared_storage.stat_tile1_empty_mbar[0]); + uint64_t *bmm_ready_mbar = + sub_tile_id == 0 ? &(shared_storage.bmm2_tile0_ready_mbar[0]) + : &(shared_storage.bmm2_tile1_ready_mbar[0]); + + wait_barrier(cast_smem_ptr_to_uint(&stat_full_mbar), stat_mbar_state); + ldtm_32dp32bit_x1(&fp32_stats[0], tmem_Stats); + + float alpha = reinterpret_cast(fp32_stats[0]); + fence_view_async_tmem_load(); + arrive_barrier(cast_smem_ptr_to_uint(stat_empty_mbar)); + + bool alpha_is_one = (alpha == 1.0f); + bool all_alpha_one = __all_sync(0xFFFFFFFF, alpha_is_one); + + float2 alpha2 = make_float2(alpha, alpha); + + wait_barrier(cast_smem_ptr_to_uint(&bmm_done_mbar), bmm_mbar_state); + + if (!all_alpha_one) { +#pragma unroll + for (int block = 0; block < TILE_O_1 / local_correction_block_size; + block++) { +// LDTM +#pragma unroll + for (int ldtm_step = 0; + ldtm_step < local_correction_block_size / 8; ldtm_step++) { + ldtm_32dp32bit_x8(&fp32_O[8 * ldtm_step], + tmem_fp32_O + + block * local_correction_block_size + + ldtm_step * 8); + } + +#pragma unroll + for (int i = 0; i < local_correction_block_size; i += 2) { + float2 in = + make_float2(reinterpret_cast(fp32_O[i + 0]), + reinterpret_cast(fp32_O[i + 1])); + float2 out = fmul2(alpha2, in); + reinterpret_cast(fp32_O[i + 0]) = out.x; + reinterpret_cast(fp32_O[i + 1]) = out.y; + } + +#pragma unroll + for (int sttm_step = 0; + sttm_step < local_correction_block_size / 8; sttm_step++) { + sttm_32dp32bit_x8(tmem_fp32_O + + block * local_correction_block_size + + sttm_step * 8, + &fp32_O[8 * sttm_step]); + } + } + } + fence_view_async_tmem_store(); + arrive_barrier(cast_smem_ptr_to_uint(bmm_ready_mbar)); + } + stat_mbar_state ^= 1; + bmm_mbar_state ^= 1; + } // End of mainloop + + // Epilogue + { + epilogue_state ^= 1; + + static constexpr int stg_step = 16 / sizeof(ElementO); + const int local_row = (tid % 8) / 1; + + // global_store_op 13 decls + int oob_M_13 = desc_7.dims[2]; + int row_13 = 0; + char *ptr_13 = + reinterpret_cast(tensor_7) + + batch_coord_1 * desc_7.strides[0] * BYTES_PER_ELEMENT_13 + + head_coord_1 * desc_7.strides[1] * BYTES_PER_ELEMENT_13 + + row_13 * desc_7.strides[2] * BYTES_PER_ELEMENT_13; + + // global_store_op 14 decls + int oob_M_14 = desc_6.dims[2]; + int row_14 = 0; + char *ptr_14 = + reinterpret_cast(tensor_6) + + batch_coord_1 * desc_6.strides[0] * BYTES_PER_ELEMENT_14 + + head_coord_1 * desc_6.strides[1] * BYTES_PER_ELEMENT_14 + + row_14 * desc_6.strides[2] * BYTES_PER_ELEMENT_14; + +#pragma unroll + for (int sub_tile_id = 0; sub_tile_id < 2; ++sub_tile_id) { + const int row_coord = q_row_coord + sub_tile_id * TILE_M_1 + tid; + ElementO *smem_O = (sub_tile_id == 0) ? shared_storage.smem_O + : shared_storage.smem_V; + + uint32_t fp32_reg_O[local_epilogue_block_size]; + uint32_t reg_O[local_epilogue_block_size / (4 / sizeof(ElementO))]; + + uint32_t tmem_Stats = sub_tile_id == 0 ? tmem_Stats_0 : tmem_Stats_1; + uint32_t tmem_fp32_O = + sub_tile_id == 0 ? tmem_fp32_O_0 : tmem_fp32_O_1; + + uint64_t &stat_full_mbar = + sub_tile_id == 0 ? shared_storage.stat_tile0_full_mbar[0] + : shared_storage.stat_tile1_full_mbar[0]; + uint64_t &bmm_done_mbar = + sub_tile_id == 0 ? shared_storage.bmm2_tile0_done_mbar[0] + : shared_storage.bmm2_tile1_done_mbar[0]; + uint64_t &stat_empty_mbar = + sub_tile_id == 0 ? shared_storage.stat_tile0_empty_mbar[0] + : shared_storage.stat_tile1_empty_mbar[0]; + uint64_t &final_stat_empty_mbar = + sub_tile_id == 0 ? shared_storage.final_stat_tile0_empty_mbar[0] + : shared_storage.final_stat_tile1_empty_mbar[0]; + + wait_barrier(cast_smem_ptr_to_uint(&stat_full_mbar), stat_mbar_state); + ldtm_32dp32bit_x2(&fp32_stats[0], tmem_Stats); + + float total_max = reinterpret_cast(fp32_stats[0]) * ln2; + float total_sum = reinterpret_cast(fp32_stats[1]); + fence_view_async_tmem_load(); + arrive_barrier(cast_smem_ptr_to_uint(&stat_empty_mbar)); + arrive_barrier(cast_smem_ptr_to_uint(&final_stat_empty_mbar)); + + float softmax_stats = + total_sum == 0.f ? NEG_INFINITY : total_max + __logf(total_sum); + + // global_store_op 13 code + if (row_coord < actual_seqlen_q_1) { + stg_32(ptr_13 + + row_coord * desc_7.strides[2] * BYTES_PER_ELEMENT_13, + &(reinterpret_cast(total_max))); + } + + // global_store_op 14 code + if (row_coord < actual_seqlen_q_1) { + stg_32(ptr_14 + + row_coord * desc_6.strides[2] * BYTES_PER_ELEMENT_14, + &(reinterpret_cast(total_sum))); + } + + float threshold_beta = (total_sum == 0.f) ? 0.f : 1.f / total_sum; + float beta = threshold_beta; + + float2 beta2 = make_float2(threshold_beta, threshold_beta); + + wait_barrier(cast_smem_ptr_to_uint(&bmm_done_mbar), bmm_mbar_state); + + if (sub_tile_id == 0) { + wait_barrier( + cast_smem_ptr_to_uint(&shared_storage.tma_o_0_empty_mbar[0]), + epilogue_state); + } +#pragma unroll + for (int block = 0; block < TILE_O_1 / local_epilogue_block_size; + block++) { + { +#pragma unroll + for (int ldtm_step = 0; + ldtm_step < local_epilogue_block_size / 32; ldtm_step++) { + ldtm_32dp32bit_x32(&fp32_reg_O[32 * ldtm_step], + tmem_fp32_O + + block * local_epilogue_block_size + + ldtm_step * 32); + } + } + +#pragma unroll + for (int i = 0; i < local_epilogue_block_size; i += 2) { + float2 in = + make_float2(reinterpret_cast(fp32_reg_O[i + 0]), + reinterpret_cast(fp32_reg_O[i + 1])); + float2 out = fmul2(beta2, in); + reinterpret_cast(fp32_reg_O[i + 0]) = out.x; + reinterpret_cast(fp32_reg_O[i + 1]) = out.y; + } + +// general_data_type_cast code +#pragma unroll + for (int i = 0; i < local_epilogue_block_size; i += 2) { + fp32x2_to_bf16x2(®_O[i / 2], &fp32_reg_O[i]); + } + +#pragma unroll + for (int i = 0; i < local_epilogue_block_size / stg_step; i++) { + char *smem_loc = get_smem_loc_epilogue_swizzle_128b( + reinterpret_cast(smem_O), block, tid, local_row, i, + sizeof(ElementO), local_epilogue_block_size, TILE_M_1); +)KERNEL" + R"KERNEL( + sts_128(cast_smem_ptr_to_uint(smem_loc), + reinterpret_cast(®_O[i * 4])); + } + + uint64_t *tma_o_full_mbar = + sub_tile_id == 0 ? &(shared_storage.tma_o_0_full_mbar[block]) + : &(shared_storage.tma_o_1_full_mbar[block]); + fence_view_async_shared(); + arrive_barrier(cast_smem_ptr_to_uint(tma_o_full_mbar)); + } + } + stat_mbar_state ^= 1; + bmm_mbar_state ^= 1; + } // Epilogue end + + // Get next tile id + wait_barrier(cast_smem_ptr_to_uint( + &shared_storage.scheduler_mbar[sched_state.index()]), + sched_state.phase()); + uint32_t tile_id_smem_int_ptr = cast_smem_ptr_to_uint( + &shared_storage.tile_id[sched_state.index() * 8]); + lds_128(tile_id_and_coord, tile_id_smem_int_ptr); + ++sched_state; + + uint32_t tile_idx = tile_id_and_coord[0]; + uint32_t bidhb, l2_mod; + uint32_t block = 0, bidhb_residual = 0; + + fastDivMod(l2_major_divmod, tile_idx, bidhb, l2_mod); + if (bidhb < num_hb_quotient) { + { + fastDivMod(l2_minor_divmod, l2_mod, block, bidhb_residual); + } + } else { + { + fastDivMod(l2_minor_residual_divmod, l2_mod, block, bidhb_residual); + } + } + + int bidhb_actual = bidhb * l2_minor_divmod.val + bidhb_residual; + uint32_t batch_idx, head_idx; + fastDivMod(num_head_divmod, bidhb_actual, batch_idx, head_idx); + block = num_block - 1 - block; + + lean_tile_id = tile_id_and_coord[0]; + + blocked_row_coord = block + 0; + head_coord_from_grid_1 = (head_idx) + 0; + batch_coord_1 = batch_idx; + is_valid_tile = tile_id_and_coord[2] & 1; + + head_coord_1 = head_coord_from_grid_1; + head_coord_k_1 = head_coord_1 / attnDesc.q_heads_per_k; + head_coord_v_1 = head_coord_1 / attnDesc.q_heads_per_v; + + q_row_coord = (blocked_row_coord * Tiles_Q * BMM1_TILE_M); + int2 kv_loop_bounds = compute_kv_loop_bounds( + q_row_coord, BMM1_TILE_M * Tiles_Q * CGA_M_1, TILE_N_1, + actual_seqlen_kv_1, actual_seqlen_q_1, shift_right_bound, left_bound); + kv_loop_left_bound = kv_loop_bounds.x; + kv_loop_right_bound = kv_loop_bounds.y; + if (actual_seqlen_q_1 == 0 || q_row_coord >= actual_seqlen_q_1) { + kv_loop_right_bound = kv_loop_left_bound; + } + } // end of persistent loop + arrive_barrier( + cast_smem_ptr_to_uint(&shared_storage.tmem_dealloc_smem_bar[0])); + } + + // thread_specialize_op 23 code + else if (wid == SoftmaxWarps + CorrectionWarps) { + // MMA + reg_dealloc(); + tmem_allocate_1sm(num_columns_per_tmem, + cast_smem_ptr_to_uint(&shared_storage.tmem_base_ptrs)); + tmem_release_allocation_lock_1sm(); + __syncwarp(); + named_barrier_arrive(TMEM_ALLOC_SOFTMAX_BARRIER, + threads_per_warp * (SoftmaxWarps + 1)); + named_barrier_arrive(TMEM_ALLOC_CORRECTION_BARRIER, + threads_per_warp * (CorrectionWarps + 1)); + uint32_t base_tmem_addr; + lds_32(&base_tmem_addr, + cast_smem_ptr_to_uint(&shared_storage.tmem_base_ptrs)); + + uint32_t tmem_S_acc_0 = base_tmem_addr + 0; + uint32_t tmem_S_acc_1 = base_tmem_addr + BMM1_TILE_N; + + uint32_t tmem_S_bmm2_0 = base_tmem_addr + BMM1_TILE_N / 2; + uint32_t tmem_S_bmm2_1 = base_tmem_addr + 3 * BMM1_TILE_N / 2; + + uint32_t tmem_O_0 = base_tmem_addr + 256; + uint32_t tmem_O_1 = base_tmem_addr + 384; + + // Memory barrier states + uint32_t bmm_mbar_state = 0; + uint32_t final_stat_mbar_state = 1; + uint32_t empty_mainloop_mbar_state = 0; + + PipelineState kv_mbar_state(0, 0); // index, state + PipelineState q_mbar_state(0, 0); // index, state + + // mma_op 8 decls + constexpr uint64_t utcmma_instruction_desc_8 = + build_utcmma_instruction_desc<0, // max shift for WS + TILE_M_1 * CTA_MMA_1, TILE_N_1, + 0, // Transpose for B + 0, // Transpose for A + 0, // Negate for B + 0, // Negate for A + 1, // Data type B + 1, // Data type A + 0, // sparse meta data format + 1, // Acc type fp32 + 0, // saturate disable + 0, // sparse disable + 0 // sparse meta data id2 ??? + >(); + // Swizzle mode, base smem offset, desc type, bytes per leading dim, tile k + Smem_utcmma_descriptor utcmma_smem_desc_Q_0(2, 0, 1, 16, + ROWS_PER_CORE_MATRIX_A_8 * 128); + Smem_utcmma_descriptor utcmma_smem_desc_K(2, 0, 1, 16, + ROWS_PER_CORE_MATRIX_B_8 * 128); + + utcmma_smem_desc_Q_0.set_smem( + cast_smem_ptr_to_uint(shared_storage.smem_Q)); + utcmma_smem_desc_K.set_smem( + cast_smem_ptr_to_uint(shared_storage.smem_K)); + + // mma_op 17 decls + constexpr uint64_t utcmma_instruction_desc_17 = + build_utcmma_instruction_desc<0, // max shift for WS + TILE_M_1 * CTA_MMA_1, TILE_O_1, + 1, // Transpose for B + 0, // Transpose for A + 0, // Negate for B + 0, // Negate for A + 1, // Data type B + 1, // Data type A + 0, // sparse meta data format + 1, // Acc type fp32 + 0, // saturate disable + 0, // sparse disable + 0 // sparse meta data id2 ??? + >(); + // Swizzle mode, base smem offset, desc type, bytes per leading dim, tile k + Smem_utcmma_descriptor utcmma_smem_desc_V( + 2, 0, 1, TILE_O_1 / ROWS_PER_CORE_MATRIX_B_17 <= 8 ? 0 : TILE_N_1 * 128, + COLS_PER_CORE_MATRIX_B_17 * 128); + + utcmma_smem_desc_V.set_smem( + cast_smem_ptr_to_uint(shared_storage.smem_V)); + +// Persistent loop over tiles +#pragma unroll 1 + while (is_valid_tile) { + if (elect_one) { + arrive_barrier(cast_smem_ptr_to_uint( + &(shared_storage.read_tile_id_done_mbar[sched_state.index()]))); + } + //---------------------------------------------------------// + //------------------- MMA Pipeline Begin ------------------// + //---------------------------------------------------------// + + if (kv_loop_left_bound < kv_loop_right_bound) { + wait_barrier(cast_smem_ptr_to_uint( + &shared_storage.final_stat_tile0_empty_mbar[0]), + final_stat_mbar_state); + + const int kv_stage = kv_mbar_state.index(); + const int kv_state = kv_mbar_state.phase(); + wait_barrier( + cast_smem_ptr_to_uint(&shared_storage.tma_k_full_mbar[kv_stage]), + kv_state); + + const int q_stage = q_mbar_state.index(); + const int q_phase = q_mbar_state.phase(); + wait_barrier( + cast_smem_ptr_to_uint(&shared_storage.tma_q_full_mbar[q_stage]), + q_phase); + ++q_mbar_state; + + // Q * K matmul for sub-tile 0 + { + Smem_utcmma_descriptor &utcmma_smem_desc_Q = utcmma_smem_desc_Q_0; + uint32_t tmem_acc = tmem_S_acc_0; + + // mma_op 8 code +#pragma unroll + for (int k = 0; k < BMM1_XMMAS_K; ++k) { + if (elect_one) + utcmma_asmem_bsmem_h_cta1(utcmma_smem_desc_Q.desc, + utcmma_smem_desc_K.desc, tmem_acc, + utcmma_instruction_desc_8, (k > 0)); + if ((k % 4) == ((BMM1_XMMAS_K - 1) % 4)) { + utcmma_smem_desc_Q.add_smem_offset<-BYTES_PER_MMA_K *( + (BMM1_XMMAS_K - 1) % 4)>(); + utcmma_smem_desc_Q.add_smem_offset<128 * BMM1_TILE_M>(); + } else { + utcmma_smem_desc_Q.add_smem_offset(); + } + if ((k % 4) == ((BMM1_XMMAS_K - 1) % 4)) { + utcmma_smem_desc_K.add_smem_offset<-BYTES_PER_MMA_K *( + (BMM1_XMMAS_K - 1) % 4)>(); + utcmma_smem_desc_K + .add_smem_offset<128 * BMM1_TILE_N / CTA_MMA_1>(); + } else { + utcmma_smem_desc_K.add_smem_offset(); + } + } + utcmma_smem_desc_Q.add_smem_offset<-128 * BMM1_TILE_M * + FORT_MAX(BMM1_XMMAS_K / 4, 1)>(); + utcmma_smem_desc_K.add_smem_offset<-128 * BMM1_TILE_N / CTA_MMA_1 * + FORT_MAX(BMM1_XMMAS_K / 4, 1)>(); + // change the smem buffer for Q + utcmma_smem_desc_Q + .increment_smem_buffer(); + } + + if (elect_one) { + umma_arrive( + cast_smem_ptr_to_uint(&shared_storage.bmm1_tile0_done_mbar[0])); + } // elect_one + } // if (kv_loop_left_bound < kv_loop_right_bound) + if (kv_loop_left_bound < kv_loop_right_bound) { + wait_barrier(cast_smem_ptr_to_uint( + &shared_storage.final_stat_tile1_empty_mbar[0]), + final_stat_mbar_state); + + const int kv_stage = kv_mbar_state.index(); + const int kv_state = kv_mbar_state.phase(); + const int q_stage = q_mbar_state.index(); + const int q_phase = q_mbar_state.phase(); + wait_barrier( + cast_smem_ptr_to_uint(&shared_storage.tma_q_full_mbar[q_stage]), + q_phase); + ++q_mbar_state; + + // Q * K matmul for sub-tile 1 + { + Smem_utcmma_descriptor &utcmma_smem_desc_Q = utcmma_smem_desc_Q_0; + uint32_t tmem_acc = tmem_S_acc_1; + + // mma_op 8 code +#pragma unroll + for (int k = 0; k < BMM1_XMMAS_K; ++k) { + if (elect_one) + utcmma_asmem_bsmem_h_cta1(utcmma_smem_desc_Q.desc, +)KERNEL" + R"KERNEL( + utcmma_smem_desc_K.desc, tmem_acc, + utcmma_instruction_desc_8, (k > 0)); + if ((k % 4) == ((BMM1_XMMAS_K - 1) % 4)) { + utcmma_smem_desc_Q.add_smem_offset<-BYTES_PER_MMA_K *( + (BMM1_XMMAS_K - 1) % 4)>(); + utcmma_smem_desc_Q.add_smem_offset<128 * BMM1_TILE_M>(); + } else { + utcmma_smem_desc_Q.add_smem_offset(); + } + if ((k % 4) == ((BMM1_XMMAS_K - 1) % 4)) { + utcmma_smem_desc_K.add_smem_offset<-BYTES_PER_MMA_K *( + (BMM1_XMMAS_K - 1) % 4)>(); + utcmma_smem_desc_K + .add_smem_offset<128 * BMM1_TILE_N / CTA_MMA_1>(); + } else { + utcmma_smem_desc_K.add_smem_offset(); + } + } + utcmma_smem_desc_Q.add_smem_offset<-128 * BMM1_TILE_M * + FORT_MAX(BMM1_XMMAS_K / 4, 1)>(); + utcmma_smem_desc_K.add_smem_offset<-128 * BMM1_TILE_N / CTA_MMA_1 * + FORT_MAX(BMM1_XMMAS_K / 4, 1)>(); + // change the smem buffer for K + utcmma_smem_desc_K + .increment_smem_buffer(); + // change the smem buffer for Q + utcmma_smem_desc_Q + .increment_smem_buffer(); + const int kv_stage = kv_mbar_state.index(); + if (elect_one) + umma_arrive(cast_smem_ptr_to_uint( + &shared_storage.tma_k_empty_mbar[kv_stage])); + } + + if (elect_one) { + umma_arrive( + cast_smem_ptr_to_uint(&shared_storage.bmm1_tile1_done_mbar[0])); + } // elect_one + } + +// Mainloop +#pragma unroll 1 + for (int kv_loop = kv_loop_left_bound + 1; kv_loop < kv_loop_right_bound; + kv_loop += 1) { + const int kv_stage = kv_mbar_state.index(); + const uint32_t kv_state = kv_mbar_state.phase(); + + ++kv_mbar_state; + + wait_barrier( + cast_smem_ptr_to_uint(&shared_storage.tma_v_full_mbar[kv_stage]), + kv_state); + + // S * V matmul for sub-tile 0 + { + uint32_t bmm2_ready_mbar = + cast_smem_ptr_to_uint(&shared_storage.bmm2_tile0_ready_mbar[0]); + uint32_t tmem_O = tmem_O_0; + uint32_t tmem_S = tmem_S_bmm2_0; + uint32_t scaleC = kv_loop > kv_loop_left_bound + 1; + + // mma_op 17 code +#pragma unroll + for (int k = 0; k < BMM2_XMMAS_K; ++k) { + if (k % (BMM2_XMMAS_K / (BMM1_TILE_N / 64)) == 0) { + wait_barrier(bmm2_ready_mbar + + (k / (BMM2_XMMAS_K / (BMM1_TILE_N / 64))) * 8, + bmm_mbar_state); + } + if (elect_one) + utcmma_atmem_bsmem_h_cta1( + tmem_S + k * (UTCMMA_TILE_K / (4 / BYTES_PER_ELEMENT)), + utcmma_smem_desc_V.desc, tmem_O, utcmma_instruction_desc_17, + scaleC || k > 0); + if (k == BMM2_XMMAS_K - 1) { + utcmma_smem_desc_V + .add_smem_offset<-UTCMMA_TILE_K * 128 * (BMM2_XMMAS_K - 1)>(); + } else { + utcmma_smem_desc_V.add_smem_offset(); + } + } + if (elect_one) + umma_arrive( + cast_smem_ptr_to_uint(&shared_storage.bmm2_tile0_done_mbar[0])); + } + + wait_barrier( + cast_smem_ptr_to_uint( + &shared_storage.tma_k_full_mbar[kv_mbar_state.index()]), + kv_mbar_state.phase()); + + // Q * K matmul for sub-tile 0 + { + Smem_utcmma_descriptor &utcmma_smem_desc_Q = utcmma_smem_desc_Q_0; + uint32_t tmem_acc = tmem_S_acc_0; + + // mma_op 8 code +#pragma unroll + for (int k = 0; k < BMM1_XMMAS_K; ++k) { + if (elect_one) + utcmma_asmem_bsmem_h_cta1(utcmma_smem_desc_Q.desc, + utcmma_smem_desc_K.desc, tmem_acc, + utcmma_instruction_desc_8, (k > 0)); + if ((k % 4) == ((BMM1_XMMAS_K - 1) % 4)) { + utcmma_smem_desc_Q.add_smem_offset<-BYTES_PER_MMA_K *( + (BMM1_XMMAS_K - 1) % 4)>(); + utcmma_smem_desc_Q.add_smem_offset<128 * BMM1_TILE_M>(); + } else { + utcmma_smem_desc_Q.add_smem_offset(); + } + if ((k % 4) == ((BMM1_XMMAS_K - 1) % 4)) { + utcmma_smem_desc_K.add_smem_offset<-BYTES_PER_MMA_K *( + (BMM1_XMMAS_K - 1) % 4)>(); + utcmma_smem_desc_K + .add_smem_offset<128 * BMM1_TILE_N / CTA_MMA_1>(); + } else { + utcmma_smem_desc_K.add_smem_offset(); + } + } + utcmma_smem_desc_Q.add_smem_offset<-128 * BMM1_TILE_M * + FORT_MAX(BMM1_XMMAS_K / 4, 1)>(); + utcmma_smem_desc_K.add_smem_offset<-128 * BMM1_TILE_N / CTA_MMA_1 * + FORT_MAX(BMM1_XMMAS_K / 4, 1)>(); + // change the smem buffer for Q + utcmma_smem_desc_Q + .increment_smem_buffer(); + } + + if (elect_one) { + umma_arrive( + cast_smem_ptr_to_uint(&shared_storage.bmm1_tile0_done_mbar[0])); + } // elect_one + + // S * V matmul for sub-tile 1 + { + uint32_t bmm2_ready_mbar = + cast_smem_ptr_to_uint(&shared_storage.bmm2_tile1_ready_mbar[0]); + uint32_t tmem_O = tmem_O_1; + uint32_t tmem_S = tmem_S_bmm2_1; + uint32_t scaleC = kv_loop > kv_loop_left_bound + 1; + + // mma_op 17 code +#pragma unroll + for (int k = 0; k < BMM2_XMMAS_K; ++k) { + if (k % (BMM2_XMMAS_K / (BMM1_TILE_N / 64)) == 0) { + wait_barrier(bmm2_ready_mbar + + (k / (BMM2_XMMAS_K / (BMM1_TILE_N / 64))) * 8, + bmm_mbar_state); + } + if (elect_one) + utcmma_atmem_bsmem_h_cta1( + tmem_S + k * (UTCMMA_TILE_K / (4 / BYTES_PER_ELEMENT)), + utcmma_smem_desc_V.desc, tmem_O, utcmma_instruction_desc_17, + scaleC || k > 0); + if (k == BMM2_XMMAS_K - 1) { + utcmma_smem_desc_V + .add_smem_offset<-UTCMMA_TILE_K * 128 * (BMM2_XMMAS_K - 1)>(); + } else { + utcmma_smem_desc_V.add_smem_offset(); + } + } + if (elect_one) + umma_arrive( + cast_smem_ptr_to_uint(&shared_storage.bmm2_tile1_done_mbar[0])); + if (elect_one) + umma_arrive(cast_smem_ptr_to_uint( + &shared_storage.tma_v_empty_mbar[kv_stage])); + // Change the buffer for V + utcmma_smem_desc_V + .increment_smem_buffer(); + } + bmm_mbar_state ^= 1; + + // Q * K matmul for sub-tile 1 + { + Smem_utcmma_descriptor &utcmma_smem_desc_Q = utcmma_smem_desc_Q_0; + uint32_t tmem_acc = tmem_S_acc_1; + + // mma_op 8 code +#pragma unroll + for (int k = 0; k < BMM1_XMMAS_K; ++k) { + if (elect_one) + utcmma_asmem_bsmem_h_cta1(utcmma_smem_desc_Q.desc, + utcmma_smem_desc_K.desc, tmem_acc, + utcmma_instruction_desc_8, (k > 0)); + if ((k % 4) == ((BMM1_XMMAS_K - 1) % 4)) { + utcmma_smem_desc_Q.add_smem_offset<-BYTES_PER_MMA_K *( + (BMM1_XMMAS_K - 1) % 4)>(); + utcmma_smem_desc_Q.add_smem_offset<128 * BMM1_TILE_M>(); + } else { + utcmma_smem_desc_Q.add_smem_offset(); + } + if ((k % 4) == ((BMM1_XMMAS_K - 1) % 4)) { + utcmma_smem_desc_K.add_smem_offset<-BYTES_PER_MMA_K *( + (BMM1_XMMAS_K - 1) % 4)>(); + utcmma_smem_desc_K + .add_smem_offset<128 * BMM1_TILE_N / CTA_MMA_1>(); + } else { + utcmma_smem_desc_K.add_smem_offset(); + } + } + utcmma_smem_desc_Q.add_smem_offset<-128 * BMM1_TILE_M * + FORT_MAX(BMM1_XMMAS_K / 4, 1)>(); + utcmma_smem_desc_K.add_smem_offset<-128 * BMM1_TILE_N / CTA_MMA_1 * + FORT_MAX(BMM1_XMMAS_K / 4, 1)>(); + // change the smem buffer for K + utcmma_smem_desc_K + .increment_smem_buffer(); + // change the smem buffer for Q + utcmma_smem_desc_Q + .increment_smem_buffer(); + const int kv_stage = kv_mbar_state.index(); + if (elect_one) + umma_arrive(cast_smem_ptr_to_uint( + &shared_storage.tma_k_empty_mbar[kv_stage])); + } + + if (elect_one) { + umma_arrive( + cast_smem_ptr_to_uint(&shared_storage.bmm1_tile1_done_mbar[0])); + } // elect_one + } + if (kv_loop_left_bound < kv_loop_right_bound) { + if (elect_one) { + umma_arrive( + cast_smem_ptr_to_uint(&shared_storage.tma_q_empty_mbar[1])); + umma_arrive( + cast_smem_ptr_to_uint(&shared_storage.tma_q_empty_mbar[0])); + } + const int kv_stage = kv_mbar_state.index(); + const uint32_t kv_state = kv_mbar_state.phase(); + ++kv_mbar_state; + + wait_barrier( + cast_smem_ptr_to_uint(&shared_storage.tma_v_full_mbar[kv_stage]), + kv_state); + + // S * V matmul for sub-tile 0 + { + uint32_t bmm2_ready_mbar = + cast_smem_ptr_to_uint(&shared_storage.bmm2_tile0_ready_mbar[0]); + uint32_t tmem_O = tmem_O_0; + uint32_t tmem_S = tmem_S_bmm2_0; + uint32_t scaleC = kv_loop_right_bound > kv_loop_left_bound + 1; + + // mma_op 17 code +#pragma unroll + for (int k = 0; k < BMM2_XMMAS_K; ++k) { + if (k % (BMM2_XMMAS_K / (BMM1_TILE_N / 64)) == 0) { + wait_barrier(bmm2_ready_mbar + +)KERNEL" + R"KERNEL( + (k / (BMM2_XMMAS_K / (BMM1_TILE_N / 64))) * 8, + bmm_mbar_state); + } + if (elect_one) + utcmma_atmem_bsmem_h_cta1( + tmem_S + k * (UTCMMA_TILE_K / (4 / BYTES_PER_ELEMENT)), + utcmma_smem_desc_V.desc, tmem_O, utcmma_instruction_desc_17, + scaleC || k > 0); + if (k == BMM2_XMMAS_K - 1) { + utcmma_smem_desc_V + .add_smem_offset<-UTCMMA_TILE_K * 128 * (BMM2_XMMAS_K - 1)>(); + } else { + utcmma_smem_desc_V.add_smem_offset(); + } + } + if (elect_one) + umma_arrive( + cast_smem_ptr_to_uint(&shared_storage.bmm2_tile0_done_mbar[0])); + } + // S * V matmul for sub-tile 1 + { + uint32_t bmm2_ready_mbar = + cast_smem_ptr_to_uint(&shared_storage.bmm2_tile1_ready_mbar[0]); + uint32_t tmem_O = tmem_O_1; + uint32_t tmem_S = tmem_S_bmm2_1; + uint32_t scaleC = kv_loop_right_bound > kv_loop_left_bound + 1; + + // mma_op 17 code +#pragma unroll + for (int k = 0; k < BMM2_XMMAS_K; ++k) { + if (k % (BMM2_XMMAS_K / (BMM1_TILE_N / 64)) == 0) { + wait_barrier(bmm2_ready_mbar + + (k / (BMM2_XMMAS_K / (BMM1_TILE_N / 64))) * 8, + bmm_mbar_state); + } + if (elect_one) + utcmma_atmem_bsmem_h_cta1( + tmem_S + k * (UTCMMA_TILE_K / (4 / BYTES_PER_ELEMENT)), + utcmma_smem_desc_V.desc, tmem_O, utcmma_instruction_desc_17, + scaleC || k > 0); + if (k == BMM2_XMMAS_K - 1) { + utcmma_smem_desc_V + .add_smem_offset<-UTCMMA_TILE_K * 128 * (BMM2_XMMAS_K - 1)>(); + } else { + utcmma_smem_desc_V.add_smem_offset(); + } + } + if (elect_one) + umma_arrive( + cast_smem_ptr_to_uint(&shared_storage.bmm2_tile1_done_mbar[0])); + if (elect_one) + umma_arrive(cast_smem_ptr_to_uint( + &shared_storage.tma_v_empty_mbar[kv_stage])); + // Change the buffer for V + utcmma_smem_desc_V + .increment_smem_buffer(); + } + bmm_mbar_state ^= 1; + } else { + wait_barrier( + cast_smem_ptr_to_uint(&shared_storage.empty_mainloop_mbar[0]), + empty_mainloop_mbar_state); + empty_mainloop_mbar_state ^= 1; + if (elect_one) { + umma_arrive( + cast_smem_ptr_to_uint(&shared_storage.bmm2_tile0_done_mbar[0])); + umma_arrive( + cast_smem_ptr_to_uint(&shared_storage.bmm2_tile1_done_mbar[0])); + } + } + final_stat_mbar_state ^= 1; + __syncwarp(); + + // Get next tile id + wait_barrier(cast_smem_ptr_to_uint( + &shared_storage.scheduler_mbar[sched_state.index()]), + sched_state.phase()); + uint32_t tile_id_smem_int_ptr = cast_smem_ptr_to_uint( + &shared_storage.tile_id[sched_state.index() * 8]); + lds_128(tile_id_and_coord, tile_id_smem_int_ptr); + ++sched_state; + + uint32_t tile_idx = tile_id_and_coord[0]; + uint32_t bidhb, l2_mod; + uint32_t block = 0, bidhb_residual = 0; + + fastDivMod(l2_major_divmod, tile_idx, bidhb, l2_mod); + if (bidhb < num_hb_quotient) { + { + fastDivMod(l2_minor_divmod, l2_mod, block, bidhb_residual); + } + } else { + { + fastDivMod(l2_minor_residual_divmod, l2_mod, block, bidhb_residual); + } + } + + int bidhb_actual = bidhb * l2_minor_divmod.val + bidhb_residual; + uint32_t batch_idx, head_idx; + fastDivMod(num_head_divmod, bidhb_actual, batch_idx, head_idx); + block = num_block - 1 - block; + + lean_tile_id = tile_id_and_coord[0]; + + blocked_row_coord = block + 0; + head_coord_from_grid_1 = (head_idx) + 0; + batch_coord_1 = batch_idx; + is_valid_tile = tile_id_and_coord[2] & 1; + + head_coord_1 = head_coord_from_grid_1; + head_coord_k_1 = head_coord_1 / attnDesc.q_heads_per_k; + head_coord_v_1 = head_coord_1 / attnDesc.q_heads_per_v; + + q_row_coord = (blocked_row_coord * Tiles_Q * BMM1_TILE_M); + int2 kv_loop_bounds = compute_kv_loop_bounds( + q_row_coord, BMM1_TILE_M * Tiles_Q * CGA_M_1, TILE_N_1, + actual_seqlen_kv_1, actual_seqlen_q_1, shift_right_bound, left_bound); + kv_loop_left_bound = kv_loop_bounds.x; + kv_loop_right_bound = kv_loop_bounds.y; + if (actual_seqlen_q_1 == 0 || q_row_coord >= actual_seqlen_q_1) { + kv_loop_right_bound = kv_loop_left_bound; + } + } + wait_barrier( + cast_smem_ptr_to_uint(&shared_storage.tmem_dealloc_smem_bar[0]), 0); + tmem_free_1sm(num_columns_per_tmem, shared_storage.tmem_base_ptrs); + } + + // thread_specialize_op 24 code + else if (wid == SoftmaxWarps + CorrectionWarps + 1) { + // TMALDG Q, K, V + reg_dealloc(); + + // Memory barrier states + uint32_t epilogue_state = 1; + PipelineState q_mbar_state(0, 1); // index, state + PipelineState kv_mbar_state(0, 1); // index, state + PipelineState<2> sts_page_buffer_state(0, 0); // index, state + PipelineState<2> lds_page_buffer_state(0, 0); // index, state + +// Persistent loop over tiles +#pragma unroll 1 + while (is_valid_tile) { + if (elect_one) { + arrive_barrier(cast_smem_ptr_to_uint( + &(shared_storage.read_tile_id_done_mbar[sched_state.index()]))); + } + + // global_load_shared_store_op 4 decls + + // global_load_shared_store_op 7 decls + + // global_load_shared_store_op 16 decls + if (kv_loop_left_bound < kv_loop_right_bound) { + int kv_loop = kv_loop_left_bound; + + const int q_stage = q_mbar_state.index(); + const int q_phase = q_mbar_state.phase(); + wait_barrier( + cast_smem_ptr_to_uint(&shared_storage.tma_q_empty_mbar[q_stage]), + q_phase); + ++q_mbar_state; + + // Load Q sub-tile 0 + if (true) { + smem_bar_set_transaction_count( + cast_smem_ptr_to_uint(&shared_storage.tma_q_full_mbar[q_stage]), + qTmaTransactionBytes, elect_one); + } + { + uint32_t local_smem_q = cast_smem_ptr_to_uint(shared_storage.smem_Q + + q_stage * qBufferElems); + uint32_t local_smem_bar_tma_q = + cast_smem_ptr_to_uint(&shared_storage.tma_q_full_mbar[q_stage]); + const uint32_t row_coord = q_row_coord; + + // global_load_shared_store_op 4 code +#pragma unroll + for (int i = 0; i < TILE_O_1 * BYTES_PER_ELEMENT_1; i += 128) { + utmaldg_4d_tiled(&tma_tensor_3, local_smem_q + i * TILE_M_1, + local_smem_bar_tma_q, i / BYTES_PER_ELEMENT_1, + row_coord, head_coord_1 + 0, batch_coord_1, + elect_one); + } + } + + // Load K + const int k_stage = kv_mbar_state.index(); + const int k_phase = kv_mbar_state.phase(); + wait_barrier( + cast_smem_ptr_to_uint(&shared_storage.tma_k_empty_mbar[k_stage]), + k_phase); + + if (true) { + smem_bar_set_transaction_count( + cast_smem_ptr_to_uint(&shared_storage.tma_k_full_mbar[k_stage]), + kTmaTransactionBytes, elect_one); + } + { + const int lds_page_buffer_state_idx = lds_page_buffer_state.index(); + uint32_t local_smem_k = cast_smem_ptr_to_uint(shared_storage.smem_K + + k_stage * kBufferElems); + uint32_t local_smem_bar_tma_k = + cast_smem_ptr_to_uint(&shared_storage.tma_k_full_mbar[k_stage]); + const uint32_t p_col_coord = kv_loop * TILE_N_1; + + // global_load_shared_store_op 7 code +#pragma unroll + for (int i = 0; i < TILE_O_1 * BYTES_PER_ELEMENT_1; i += 128) { + utmaldg_4d_tiled(&tma_tensor_2, local_smem_k + i * TILE_N_1, + local_smem_bar_tma_k, i / BYTES_PER_ELEMENT_1, + p_col_coord, head_coord_k_1, batch_coord_1, + elect_one); + } + } + } // if (kv_loop_left_bound < kv_loop_right_bound) + + if (kv_loop_left_bound < kv_loop_right_bound) { + const int q_stage = q_mbar_state.index(); + const int q_phase = q_mbar_state.phase(); + wait_barrier( + cast_smem_ptr_to_uint(&shared_storage.tma_q_empty_mbar[q_stage]), + q_phase); + ++q_mbar_state; + + // Load Q sub-tile 1 + if (true) { + smem_bar_set_transaction_count( + cast_smem_ptr_to_uint(&shared_storage.tma_q_full_mbar[q_stage]), + qTmaTransactionBytes, elect_one); + } + { + uint32_t local_smem_q = cast_smem_ptr_to_uint(shared_storage.smem_Q + + q_stage * qBufferElems); + uint32_t local_smem_bar_tma_q = + cast_smem_ptr_to_uint(&shared_storage.tma_q_full_mbar[q_stage]); + const uint32_t row_coord = q_row_coord + TILE_M_1; + + // global_load_shared_store_op 4 code +#pragma unroll + for (int i = 0; i < TILE_O_1 * BYTES_PER_ELEMENT_1; i += 128) { + utmaldg_4d_tiled(&tma_tensor_3, local_smem_q + i * TILE_M_1, + local_smem_bar_tma_q, i / BYTES_PER_ELEMENT_1, + row_coord, head_coord_1 + 0, batch_coord_1, + elect_one); + } + } + } // if (kv_loop_left_bound < kv_loop_right_bound) + wait_barrier(cast_smem_ptr_to_uint(&shared_storage.epilogue_done_mbar[0]), + epilogue_state); + epilogue_state ^= 1; + if (kv_loop_left_bound < kv_loop_right_bound) { + int kv_loop = kv_loop_left_bound; + // Load V + const int v_stage = kv_mbar_state.index(); + const int v_phase = kv_mbar_state.phase(); + wait_barrier( + cast_smem_ptr_to_uint(&shared_storage.tma_v_empty_mbar[v_stage]), + v_phase); + ++kv_mbar_state; + + if (true) { + smem_bar_set_transaction_count( + cast_smem_ptr_to_uint(&shared_storage.tma_v_full_mbar[v_stage]), + vTmaTransactionBytes, elect_one); + } + { + const int lds_page_buffer_state_idx = lds_page_buffer_state.index(); + uint32_t local_smem_v = cast_smem_ptr_to_uint(shared_storage.smem_V + + v_stage * vBufferElems); + uint32_t local_smem_bar_tma_v = + cast_smem_ptr_to_uint(&shared_storage.tma_v_full_mbar[v_stage]); + const uint32_t p_col_coord = kv_loop * TILE_N_1; + + // global_load_shared_store_op 16 code +#pragma unroll + for (int i = 0; i < TILE_O_1 * BYTES_PER_ELEMENT_1; + i += 128 * CTA_MMA_1) { + utmaldg_4d_tiled(&tma_tensor_1, local_smem_v + i * TILE_N_1, + local_smem_bar_tma_v, 0 + i / BYTES_PER_ELEMENT_1, + p_col_coord, head_coord_v_1, batch_coord_1, + elect_one); + } + } + } // if (kv_loop_left_bound < kv_loop_right_bound) + arrive_barrier( + cast_smem_ptr_to_uint(&shared_storage.tmaldg_tile_started_mbar[0])); + +// Mainloop to load K and V +#pragma unroll 1 + for (int kv_loop = kv_loop_left_bound + 1; kv_loop < kv_loop_right_bound; + kv_loop += 1) { + + // Load K + const int k_stage = kv_mbar_state.index(); + const int k_phase = kv_mbar_state.phase(); + wait_barrier( + cast_smem_ptr_to_uint(&shared_storage.tma_k_empty_mbar[k_stage]), + k_phase); + +)KERNEL" + R"KERNEL( + if (true) { + smem_bar_set_transaction_count( + cast_smem_ptr_to_uint(&shared_storage.tma_k_full_mbar[k_stage]), + kTmaTransactionBytes, elect_one); + } + { + const int lds_page_buffer_state_idx = lds_page_buffer_state.index(); + uint32_t local_smem_k = cast_smem_ptr_to_uint(shared_storage.smem_K + + k_stage * kBufferElems); + uint32_t local_smem_bar_tma_k = + cast_smem_ptr_to_uint(&shared_storage.tma_k_full_mbar[k_stage]); + const uint32_t p_col_coord = kv_loop * TILE_N_1; + + // global_load_shared_store_op 7 code +#pragma unroll + for (int i = 0; i < TILE_O_1 * BYTES_PER_ELEMENT_1; i += 128) { + utmaldg_4d_tiled(&tma_tensor_2, local_smem_k + i * TILE_N_1, + local_smem_bar_tma_k, i / BYTES_PER_ELEMENT_1, + p_col_coord, head_coord_k_1, batch_coord_1, + elect_one); + } + } + if (kv_loop + 1 < kv_loop_right_bound) { + } + + // Load V + const int v_stage = kv_mbar_state.index(); + const int v_phase = kv_mbar_state.phase(); + wait_barrier( + cast_smem_ptr_to_uint(&shared_storage.tma_v_empty_mbar[v_stage]), + v_phase); + ++kv_mbar_state; + + if (true) { + smem_bar_set_transaction_count( + cast_smem_ptr_to_uint(&shared_storage.tma_v_full_mbar[v_stage]), + vTmaTransactionBytes, elect_one); + } + { + const int lds_page_buffer_state_idx = lds_page_buffer_state.index(); + uint32_t local_smem_v = cast_smem_ptr_to_uint(shared_storage.smem_V + + v_stage * vBufferElems); + uint32_t local_smem_bar_tma_v = + cast_smem_ptr_to_uint(&shared_storage.tma_v_full_mbar[v_stage]); + const uint32_t p_col_coord = kv_loop * TILE_N_1; + + // global_load_shared_store_op 16 code +#pragma unroll + for (int i = 0; i < TILE_O_1 * BYTES_PER_ELEMENT_1; + i += 128 * CTA_MMA_1) { + utmaldg_4d_tiled(&tma_tensor_1, local_smem_v + i * TILE_N_1, + local_smem_bar_tma_v, 0 + i / BYTES_PER_ELEMENT_1, + p_col_coord, head_coord_v_1, batch_coord_1, + elect_one); + } + } + + ++lds_page_buffer_state; + } + if (kv_loop_left_bound < kv_loop_right_bound) { + ++lds_page_buffer_state; + ++sts_page_buffer_state; + } + __syncwarp(); + // Get next tile id + wait_barrier(cast_smem_ptr_to_uint( + &shared_storage.scheduler_mbar[sched_state.index()]), + sched_state.phase()); + uint32_t tile_id_smem_int_ptr = cast_smem_ptr_to_uint( + &shared_storage.tile_id[sched_state.index() * 8]); + lds_128(tile_id_and_coord, tile_id_smem_int_ptr); + ++sched_state; + + uint32_t tile_idx = tile_id_and_coord[0]; + uint32_t bidhb, l2_mod; + uint32_t block = 0, bidhb_residual = 0; + + fastDivMod(l2_major_divmod, tile_idx, bidhb, l2_mod); + if (bidhb < num_hb_quotient) { + { + fastDivMod(l2_minor_divmod, l2_mod, block, bidhb_residual); + } + } else { + { + fastDivMod(l2_minor_residual_divmod, l2_mod, block, bidhb_residual); + } + } + + int bidhb_actual = bidhb * l2_minor_divmod.val + bidhb_residual; + uint32_t batch_idx, head_idx; + fastDivMod(num_head_divmod, bidhb_actual, batch_idx, head_idx); + block = num_block - 1 - block; + + lean_tile_id = tile_id_and_coord[0]; + + blocked_row_coord = block + 0; + head_coord_from_grid_1 = (head_idx) + 0; + batch_coord_1 = batch_idx; + is_valid_tile = tile_id_and_coord[2] & 1; + + head_coord_1 = head_coord_from_grid_1; + head_coord_k_1 = head_coord_1 / attnDesc.q_heads_per_k; + head_coord_v_1 = head_coord_1 / attnDesc.q_heads_per_v; + + q_row_coord = (blocked_row_coord * Tiles_Q * BMM1_TILE_M); + int2 kv_loop_bounds = compute_kv_loop_bounds( + q_row_coord, BMM1_TILE_M * Tiles_Q * CGA_M_1, TILE_N_1, + actual_seqlen_kv_1, actual_seqlen_q_1, shift_right_bound, left_bound); + kv_loop_left_bound = kv_loop_bounds.x; + kv_loop_right_bound = kv_loop_bounds.y; + if (actual_seqlen_q_1 == 0 || q_row_coord >= actual_seqlen_q_1) { + kv_loop_right_bound = kv_loop_left_bound; + } + } + } + + // thread_specialize_op 25 code + else if (wid == SoftmaxWarps + CorrectionWarps + 2) { + // TMASTG (epilogue) + reg_dealloc(); + + ElementO *smem_O_0 = shared_storage.smem_O; + ElementO *smem_O_1 = shared_storage.smem_V; + // Memory barrier states + uint32_t epilogue_state = 0; + +// Persistent loop over tiles +#pragma unroll 1 + while (is_valid_tile) { + if (elect_one) { + arrive_barrier(cast_smem_ptr_to_uint( + &(shared_storage.read_tile_id_done_mbar[sched_state.index()]))); + } + + wait_barrier( + cast_smem_ptr_to_uint(&shared_storage.tmaldg_tile_started_mbar[0]), + epilogue_state); + + // global_store_op 20 decls + +// TMASTG for sub-tile 0 +#pragma unroll + for (int i = 0; i < BMM2_TILE_N * sizeof(ElementO); i += 128) { + wait_barrier( + cast_smem_ptr_to_uint(&shared_storage.tma_o_0_full_mbar[i / 128]), + epilogue_state); + const uint32_t row_coord = q_row_coord; + uint32_t smem_loc = cast_smem_ptr_to_uint(smem_O_0) + i * BMM2_TILE_M; + if (actual_seqlen_q_1 > 0) { + + // global_store_op 20 code + utmastg_4d_tiled(&tma_tensor_5, smem_loc, 0 + i / sizeof(ElementO), + row_coord, head_coord_1 + 0, batch_coord_1); + } + } + + tmastg_arrive(); + +// TMASTG for sub-tile 1 +#pragma unroll + for (int i = 0; i < BMM2_TILE_N * sizeof(ElementO); i += 128) { + wait_barrier( + cast_smem_ptr_to_uint(&shared_storage.tma_o_1_full_mbar[i / 128]), + epilogue_state); + const uint32_t row_coord = q_row_coord + TILE_M_1; + uint32_t smem_loc = cast_smem_ptr_to_uint(smem_O_1) + i * BMM2_TILE_M; + if (actual_seqlen_q_1 > 0) { + + // global_store_op 20 code + utmastg_4d_tiled(&tma_tensor_5, smem_loc, 0 + i / sizeof(ElementO), + row_coord, head_coord_1 + 0, batch_coord_1); + } + } + tmastg_arrive(); + + tmastg_wait_count<1>(); + arrive_barrier( + cast_smem_ptr_to_uint(&(shared_storage.tma_o_0_empty_mbar[0]))); + + tmastg_wait_count<0>(); + arrive_barrier( + cast_smem_ptr_to_uint(&(shared_storage.epilogue_done_mbar[0]))); + + epilogue_state ^= 1; + + // Get next tile id + wait_barrier(cast_smem_ptr_to_uint( + &shared_storage.scheduler_mbar[sched_state.index()]), + sched_state.phase()); + uint32_t tile_id_smem_int_ptr = cast_smem_ptr_to_uint( + &shared_storage.tile_id[sched_state.index() * 8]); + lds_128(tile_id_and_coord, tile_id_smem_int_ptr); + ++sched_state; + + uint32_t tile_idx = tile_id_and_coord[0]; + uint32_t bidhb, l2_mod; + uint32_t block = 0, bidhb_residual = 0; + + fastDivMod(l2_major_divmod, tile_idx, bidhb, l2_mod); + if (bidhb < num_hb_quotient) { + { + fastDivMod(l2_minor_divmod, l2_mod, block, bidhb_residual); + } + } else { + { + fastDivMod(l2_minor_residual_divmod, l2_mod, block, bidhb_residual); + } + } + + int bidhb_actual = bidhb * l2_minor_divmod.val + bidhb_residual; + uint32_t batch_idx, head_idx; + fastDivMod(num_head_divmod, bidhb_actual, batch_idx, head_idx); + block = num_block - 1 - block; + + lean_tile_id = tile_id_and_coord[0]; + + blocked_row_coord = block + 0; + head_coord_from_grid_1 = (head_idx) + 0; + batch_coord_1 = batch_idx; + is_valid_tile = tile_id_and_coord[2] & 1; + + head_coord_1 = head_coord_from_grid_1; + head_coord_k_1 = head_coord_1 / attnDesc.q_heads_per_k; + head_coord_v_1 = head_coord_1 / attnDesc.q_heads_per_v; + + q_row_coord = (blocked_row_coord * Tiles_Q * BMM1_TILE_M); + int2 kv_loop_bounds = compute_kv_loop_bounds( + q_row_coord, BMM1_TILE_M * Tiles_Q * CGA_M_1, TILE_N_1, + actual_seqlen_kv_1, actual_seqlen_q_1, shift_right_bound, left_bound); + kv_loop_left_bound = kv_loop_bounds.x; + kv_loop_right_bound = kv_loop_bounds.y; + if (actual_seqlen_q_1 == 0 || q_row_coord >= actual_seqlen_q_1) { + kv_loop_right_bound = kv_loop_left_bound; + } + } // end of persistent loop + } + + // thread_specialize_op 26 code + else if (wid == SoftmaxWarps + CorrectionWarps + 3) { + // Scheduler + reg_dealloc(); + +#pragma unroll 1 + while (is_valid_tile) { + wait_barrier( + cast_smem_ptr_to_uint( + &shared_storage.read_tile_id_done_mbar[sched_state.index()]), + sched_state.phase()); + uint32_t tile_id_smem_int_ptr = cast_smem_ptr_to_uint( + &shared_storage.tile_id[sched_state.index() * 8]); + + smem_bar_set_transaction_count( + cast_smem_ptr_to_uint( + &shared_storage.scheduler_mbar[sched_state.index()]), + 16, elect_one); + + if (elect_one) { + get_next_block_id( + tile_id_smem_int_ptr, + cast_smem_ptr_to_uint( + &shared_storage.scheduler_mbar[sched_state.index()])); + } + __syncwarp(); + + wait_barrier(cast_smem_ptr_to_uint( + &shared_storage.scheduler_mbar[sched_state.index()]), + sched_state.phase()); + lds_128(tile_id_and_coord, tile_id_smem_int_ptr); + ++sched_state; + + is_valid_tile = tile_id_and_coord[2] & 1; + } + } +} +)KERNEL"; +inline constexpr size_t sm100_d64_fprop_source_len = sizeof(sm100_d64_fprop_source) - 1; + +inline constexpr const char sm100_d64_fprop_flags[] = R"FLAGS( +--gpu-architecture=sm_100a +--std=c++17 +-w +--define-macro=__CUDACC_RTC__ +-default-device +--use_fast_math +-Xptxas=-maxrregcount=128 +-DCUDA_ENABLE_TENSOR_MEMORY_INTRINSICS=1 +-DCUDA_ENABLE_VIRTCOUNT_INTRINSICS=1 +-DCUDA_ENABLE_TMEM_MANAGEMENT_INTRINSICS=1 +-DCUDA_BLACKWELL_TMA_SWIZZLE_ENABLED=1 +-DCUDA_ENABLE_FLEXIBLE_CLUSTER=1 +-DCUDA_PTX_TCMMA_V2_SUPPORTED=1 +-DCUDA_PTX_TMEM_MANAGEMENT_SUPPORTED=1 +-DCUDA_ENABLE_CLUSTER_MMA_INTRINSICS=1 +-DCUTLASS_ENABLE_TENSOR_CORE_MMA=1 +-DCUTLASS_ENABLE_EXTENDED_PTX=1 +-DCUTLASS_ENABLE_INTERNAL_NVVM=1 +-DCUTLASS_CUDA_INTERNAL_L2_PREFETCH_ENABLED=1 +-DCUTLASS_CUDA_RP2RP_ENABLED=1 +-DCUTLASS_ENABLE_COMPILER_KNOBS=1 +-DCUTLASS_TEST_LEVEL=0 +-DCUTLASS_TEST_ENABLE_CACHED_RESULTS=1 +-DCUTLASS_CONV_UNIT_TEST_RIGOROUS_SIZE_ENABLED=1 +-DCUTLASS_DEBUG_TRACE_LEVEL=0 +-DCUTLASS_VERSIONS_GENERATED +)FLAGS"; +inline constexpr size_t sm100_d64_fprop_flags_len = sizeof(sm100_d64_fprop_flags) - 1; + +} // namespace generated +} // namespace experimental +} // namespace cudnn_frontend +#if defined(_MSC_VER) +#pragma warning(pop) +#endif +#if defined(__clang__) +#pragma clang diagnostic pop +#endif \ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/generated/sdpa/sm90/prefill/full_seqlens/d128_fprop_kernel.h b/deps/cudnn-frontend/include/cudnn_frontend/generated/sdpa/sm90/prefill/full_seqlens/d128_fprop_kernel.h new file mode 100644 index 0000000..d588ce6 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/generated/sdpa/sm90/prefill/full_seqlens/d128_fprop_kernel.h @@ -0,0 +1,1762 @@ +// Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: MIT +#pragma once +// Suppress clang warning for overlength strings (concatenated result may exceed 65536) +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Woverlength-strings" +#endif + +// Suppress MSVC warning C4068 (unknown pragma) if clang pragmas are seen +#if defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable : 4068) +#endif + +namespace cudnn_frontend { +namespace experimental { +namespace generated { + +inline constexpr const char d128_fprop_source[] = + R"KERNEL( +//receive_op 0 includes +typedef unsigned long long uint64_t; +typedef unsigned int uint32_t; +typedef unsigned short uint16_t; +typedef unsigned char uint8_t; +typedef long long int64_t; +typedef int int32_t; +typedef short int16_t; +typedef signed char int8_t; +typedef unsigned int r32; + +struct CUtensorMap { + alignas(64) uint64_t opaque[16]; +}; + +#define CUDACC_VERSION __CUDACC_VER_MAJOR__ * 10 + __CUDACC_VER_MINOR__ + +namespace fort { + static const uint64_t AMPERE_MEM_DESC_DEFAULT = uint64_t(0x1000000000000000ul); + static const uint64_t MEM_DESC_DEFAULT = AMPERE_MEM_DESC_DEFAULT; + static const uint32_t MMA_PEER_BIT_MASK = 0xFEFFFFFF; + +#define NEG_INFINITY __int_as_float(0xff800000) + + typedef uint16_t half_t; + typedef uint16_t bfloat16_t; + +#define FORT_MIN(a,b) ((a) < (b) ? (a) : (b)) +#define FORT_MAX(a,b) ((a) > (b) ? (a) : (b)) +#define FORT_DIV_UP(a,b) (((a) + (b) - 1) / (b)) +#define FORT_ROUND_UP(a,b) ((((a) + (b) - 1) / (b)) * (b)) + + typedef struct tensor_descriptor { + static const int MAX_DIMS = 12; + + int64_t num_dims; + int64_t dims[MAX_DIMS]; + int64_t strides[MAX_DIMS]; + } tensor_descriptor; + + typedef struct FastDivisor { + uint32_t val, shr, mul; + } FastDivisor_t; + + inline __device__ void + fastDivMod(const FastDivisor_t &d, uint32_t val, uint32_t &div, uint32_t &mod) { + div = __umulhi((uint32_t)2 * val, d.mul) >> d.shr; + mod = val - div * d.val; + } + __device__ __inline__ void cfence() {} + +#if __CUDA_ARCH__ >= 900 + + inline __device__ uint32_t elect_one_sync() { + uint32_t pred = 0; + uint32_t laneid = 0; + asm volatile( + "{\n" + ".reg .b32 %%rx;\n" + ".reg .pred %%px;\n" + " elect.sync %%rx|%%px, %2;\n" + "@%%px mov.s32 %1, 1;\n" + " mov.s32 %0, %%rx;\n" + "}\n" : "+r"(laneid), "+r"(pred) : "r"(0xFFFFFFFF)); + return pred; + } + + template + inline __device__ void reg_alloc() { + // const int TARGET_REG_COUNT = 232; // Example values (use with reg_delloc's VALUE together): 224, 232, 208, 216 + asm volatile("setmaxnreg.inc.sync.aligned.u32 %0;\n" : : "n"(TARGET_REG_COUNT)); + } + + template + inline __device__ void reg_dealloc() { + // const int TARGET_REG_COUNT = 40; // Exmaple values: 56, 40, 88, 72 + asm volatile("setmaxnreg.dec.sync.aligned.u32 %0;\n" : : "n"(TARGET_REG_COUNT)); + } + + inline __device__ void named_barrier_arrive(uint32_t BARRIER_ID, uint32_t NUM_THREADS) { + asm volatile("bar.arrive %0, %1;" : : "r"(BARRIER_ID), "r"(NUM_THREADS)); + } + + inline __device__ void named_barrier_wait(uint32_t BARRIER_ID, uint32_t NUM_THREADS) { + asm volatile("bar.sync %0, %1;" :: "r"(BARRIER_ID), "r"(NUM_THREADS)); + } + + inline __device__ void tmastg_arrive() { + asm volatile("cp.async.bulk.commit_group;"); + } + + inline __device__ void tmastg_wait() { + asm volatile("cp.async.bulk.wait_group.read %0;" : :"n"(0):"memory"); + } + +#endif + +#if defined( __CUDA_ARCH__ ) && __CUDA_ARCH__ >= 900 + inline __device__ void warpgroup_arrive() { + asm volatile("wgmma.fence.sync.aligned;\n" ::: "memory"); + } + template inline __device__ void warpgroup_wait() { + asm volatile("wgmma.wait_group.sync.aligned %0;\n" ::"n"(N) : "memory"); + } + inline __device__ void warpgroup_commit() { + asm volatile("wgmma.commit_group.sync.aligned;\n" ::: "memory"); + } + template + inline __device__ T &__wgmma_fence_operand(T &®) { + asm volatile("" : "+r"(reg)::"memory"); + return reg; + } +#endif + +#if defined( __CUDA_ARCH__ ) && __CUDA_ARCH__ >= 900 + inline __device__ void smem_bar_init(uint32_t smem_ptr, uint32_t thread_count) { + asm volatile ("mbarrier.init.shared::cta.b64 [%0], %1;\n" :: "r"(smem_ptr), "r"(thread_count)); + } + + inline __device__ void smem_bar_set_transaction_count(uint32_t smem_ptr, uint32_t expected_copy_bytes, uint32_t pred = 0) { + asm volatile("{\n\t.reg .pred p;" + " \n\tsetp.eq.u32 p, %2, 1;" + " \n\t@p mbarrier.arrive.expect_tx.shared::cta.b64 _, [%0], %1;" + " \n\t}" :: "r"(smem_ptr), "r"(expected_copy_bytes), "r"(pred)); + } + + inline __device__ uint32_t smem_bar_peek(uint32_t smem_ptr, uint32_t bar_phase) { + uint32_t bar_phase_out; + asm volatile("{\n\t.reg .pred P1;" + " \n\tmbarrier.try_wait.parity.shared::cta.b64 P1, [%1], %2;" + " \n\tselp.b32 %0, 1, 0, P1; \n\t}" :"=r" (bar_phase_out):"r"(smem_ptr),"r"(bar_phase)); + return bar_phase_out; + } + + inline __device__ void smem_bar_wait(uint32_t smem_ptr, uint32_t bar_phase) { + + uint32_t large_val = 0x989680; + asm volatile("{\n\t.reg .pred P1;" + " \n\tLAB_WAIT:" + " \n\tmbarrier.try_wait.parity.shared::cta.b64 P1, [%0], %1, %2;" + " \n\t@P1 bra.uni DONE;" + " \n\tbra.uni LAB_WAIT;" + " \n\tDONE:" + " \n\t}" :: "r"(smem_ptr),"r"(bar_phase), "r"(large_val)); + + } + + inline __device__ + void + wait_barrier(uint32_t smem_int_ptr, + int phase_bit) // Current phase bit the barrier waiting to flip + { + asm volatile( + "{\n" + ".reg .pred P1;\n" + "LAB_WAIT:\n" + "mbarrier.try_wait.parity.shared::cta.b64 P1, [%0], %1;\n" + "@P1 bra.uni DONE;\n" + "bra.uni LAB_WAIT;\n" + "DONE:\n" + "}\n" + :: "r"(smem_int_ptr), + "r"(phase_bit)); + } + + inline __device__ void smem_bar_arrive(uint32_t smem_ptr) { + asm volatile("{\n\t.reg .b64 state;" + " \n\tmbarrier.arrive.shared::cta.b64 state, [%0];" + " \n\t}" :: "r"(smem_ptr)); + } + + // Barrier arrive on local smem + inline __device__ + void arrive_barrier(uint32_t smem_addr) { + asm volatile( + "{\n\t" + "mbarrier.arrive.shared::cta.b64 _, [%0];\n\t" + "}" + : + : "r"(smem_addr)); + } + +#endif + + extern "C" __device__ uint32_t __nvvm_get_smem_pointer(void *ptr); + + inline __device__ uint32_t get_smem_pointer(const void *ptr) { + return __nvvm_get_smem_pointer(const_cast(ptr)); + } + + extern "C" __device__ inline void *__nv_cvta_shared_to_generic_impl(size_t __ptr) { + return (void *)(void __attribute__((address_space(3))) *)__ptr; + } + inline __device__ void *set_smem_pointer(uint32_t ptr) { + return __nv_cvta_shared_to_generic_impl(ptr); + } + + inline __device__ uint32_t + cast_smem_ptr_to_uint(void const* const ptr) { + return static_cast(__cvta_generic_to_shared(ptr)); + } + + inline __device__ void stsm_x4(r32 dst, r32 src[4]) { + asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" ::"r"(dst), "r"(src[0]), "r"(src[1]), "r"(src[2]), "r"(src[3])); + } + + inline __device__ void fence_view_async_shared(void){ + asm volatile ("fence.proxy.async.shared::cta;\n"); + } + + inline __device__ void sts_32(r32 dst, r32 src[1]) { + asm volatile("st.shared.b32 [%0], %1;\n" :: "r"(dst), "r"(src[0])); + } + + inline __device__ void lds_32(r32 dst[1], r32 src) { + asm volatile("ld.shared.b32 %0, [%1];\n" : "=r"(dst[0]) : "r"(src)); + } + + inline __device__ void stg_32(void *ptr, r32 val[1]) { + uint32_t *p = reinterpret_cast(ptr); + p[0] = val[0]; + } + + // Forward ReLU activations + + inline __device__ uint32_t fp32_exp2(uint32_t in) { + uint32_t out; + asm volatile("ex2.approx.ftz.f32 %0, %1;\n" : "=r"(out) : "r"(in)); + return out; + } + + inline __device__ void fp32x2_to_bf16x2(r32 dst[1], const r32 src[2]) { + asm volatile("cvt.rn.bf16x2.f32 %0, %1, %2;\n" : "=r"(dst[0]) : "r"(src[1]), "r"(src[0])); + } + + inline __device__ void bf16x2_to_fp32x2(r32 dst[2], const r32 src[1]) { + asm volatile( \ + "{\n" \ + " .reg .b16 lo, hi;\n" \ + " mov.b32 {lo, hi}, %2;\n" \ + " mov.b32 %0, {0, lo};\n" \ + " mov.b32 %1, {0, hi};\n" \ + "}\n" : "=r"(dst[0]), "=r"(dst[1]) : "r"(src[0])); + } + + //mma_pipeline_op 1 includes + typedef struct AttentionDescriptor { + // Input parameters + // b - batch + // q_h - num heads of q/dq/o/do + // k_h - num heads of k/dk + // v_h - num heads of v/dv + // s_q - max sequence length of q + // s_kv - max sequence length of kv + // d - hidden dim (head dim) + uint32_t b, q_h, k_h, v_h, s_q, s_kv, d_qk, d_v; + uint16_t q_heads_per_k, q_heads_per_v, min_q_heads_per_kv; + } AttentionDescriptor_t; + + // CUtensorMap, when issued using bulk copy async functions, are cached in constant cache. + // But the producer kernel may have directly written to global memory, without invalidating this constant cache. + // This acquire fence, invalidates the memory address in constant cache, using UTMACCTL.IV. + inline __device__ void tma_descriptor_fence_acquire( + CUtensorMap const* p_desc + ) { +#if (__CUDA_ARCH__ >= 900) && (CUDACC_VERSION >= 123) + uint64_t gmem_int_desc = reinterpret_cast(p_desc); + asm volatile ( + "fence.proxy.tensormap::generic.acquire.gpu [%0], 128;\n" + : + : "l"(gmem_int_desc) + : "memory"); + asm volatile ( + "cvta.global.u64 %0, %0;\n" + : + : "l"(gmem_int_desc), "l"(gmem_int_desc) + : "memory"); +#endif + } + + inline __device__ void utmaldg_4d_tiled ( + const void *p_desc, + uint32_t urb0, // smem offset + uint32_t urb1, // smem barrier offset + int32_t urb2, // m + int32_t urb3, // n + int32_t urb4, // b + int32_t urb5, // extra coord + const uint32_t elect_one, + const uint64_t mem_desc = MEM_DESC_DEFAULT + ) { + if (elect_one) { +#if __CUDA_ARCH__ >= 1200 && __CUDA_ARCH__ < 1300 + asm volatile( + "cp.async.bulk.tensor.4d.shared::cta.global.mbarrier::complete_tx::bytes.L2::cache_hint [%0], [%1, {%2, %3, %4, %5}], [%6], %7;\n" :: + "r"(urb0), + "l"(reinterpret_cast(p_desc)), + "r"(urb2), + "r"(urb3), + "r"(urb4), + "r"(urb5), + "r"(urb1), + "l"(mem_desc) : "memory" + ); +#else + asm volatile( + "cp.async.bulk.tensor.4d.shared::cluster.global.mbarrier::complete_tx::bytes.L2::cache_hint [%0], [%1, {%2, %3, %4, %5}], [%6], %7;\n" :: + "r"(urb0), + "l"(reinterpret_cast(p_desc)), + "r"(urb2), + "r"(urb3), + "r"(urb4), + "r"(urb5), + "r"(urb1), + "l"(mem_desc) : "memory" + ); +#endif + } + } + + inline __device__ void utmastg_4d_tiled( + const void *p_desc, + uint32_t smem_offset, + int32_t urb0, // m + int32_t urb1, // n + int32_t urb2, // h + int32_t urb3, // b + uint64_t mem_desc = MEM_DESC_DEFAULT + ) { + asm volatile( + "cp.async.bulk.tensor.4d.global.shared::cta.bulk_group [%0, {%1, %2, %3, %4}], [%5];\n" :: + "l"(reinterpret_cast(p_desc)), + "r"(urb0), + "r"(urb1), + "r"(urb2), + "r"(urb3), + "r"(smem_offset) : "memory" + ); + } + + inline __device__ void ldgdepbar() { + asm volatile("cp.async.commit_group;\n" ::); + } + + template + inline __device__ void depbar() { + asm volatile("cp.async.wait_group %0;\n" :: "n"(STAGES)); + } + + //mma_loop_op 2 includes + + //global_load_shared_store_op 4 includes + + // shared_load_op 5 includes + + // shared_load_op 6 includes + + //global_load_shared_store_op 7 includes + + //mma_op 8 includes +)KERNEL" + R"KERNEL( +// === Section: GMMA Operations === + inline __device__ void bf16gmma_fp32_64x128x16_TN_asmem_bsmem_isb(const uint64_t &desc_a, const uint64_t &desc_b, r32 acc[64]) { + __wgmma_mma_async_bf16(64, 128, 16, false, false, false, false, true, acc, desc_a, desc_b); + warpgroup_commit(); + } + + // === Section: GMMA Operations === + inline __device__ void bf16gmma_fp32_64x128x16_TN_asmem_bsmem(const uint64_t &desc_a, const uint64_t &desc_b, r32 acc[64]) { + __wgmma_mma_async_bf16(64, 128, 16, false, false, false, false, true, acc, desc_a, desc_b); + } + +#define BMM_S_GMMA bf16gmma_fp32_64x128x16_TN_asmem_bsmem +#define BMM_S_GMMA_ISB bf16gmma_fp32_64x128x16_TN_asmem_bsmem_isb +#define BMM_S_GMMA_C0 bf16gmma_fp32_64x128x16_TN_asmem_bsmem_ignoreC + + //pointwise_calc_op 9 includes + + //global_load_op 10 includes + + //softmax_op 12 includes + + // global_store_op 13 includes + + // global_store_op 14 includes + + // shared_load_op 15 includes + + //global_load_shared_store_op 16 includes + + //mma_op 17 includes + inline __device__ void bf16gmma_fp32_64x128x16_TT_arf_bsmem_isb(const r32 a[4], const uint64_t &desc_b, r32 acc[64]) { + __wgmma_mma_async_bf16(64, 128, 16, false, false, true, false, true, acc, a, desc_b); + warpgroup_commit(); + } + + inline __device__ void bf16gmma_fp32_64x128x16_TT_arf_bsmem(const r32 a[4], const uint64_t &desc_b, r32 acc[64]) { + __wgmma_mma_async_bf16(64, 128, 16, false, false, true, false, true, acc, a, desc_b); + } + +#define BMM_O_GMMA bf16gmma_fp32_64x128x16_TT_arf_bsmem +#define BMM_O_GMMA_ISB bf16gmma_fp32_64x128x16_TT_arf_bsmem_isb +#define BMM_O_GMMA_C0 bf16gmma_fp32_64x128x16_TT_arf_bsmem_ignoreC + + // global_store_op 20 includes + + //output_loop_op 3 includes + + // shared_store_op 18 includes + + // shared_load_op 19 includes +} // namespace fort + +//receive_op 0 types +using namespace fort; + +static constexpr int THREADS_PER_WARP_0 = 32; +static constexpr int WARPS_PER_GROUP_0 = FORT_MIN(4, 384 / THREADS_PER_WARP_0); +static constexpr int THREADS_PER_WARP_GROUP_0 = THREADS_PER_WARP_0 * WARPS_PER_GROUP_0; +static constexpr int BITS_PER_REGISTER_0 = 32; // NOTE: sizeof(uint) +static constexpr int BYTES_PER_REGISTER_0 = BITS_PER_REGISTER_0 / 8; +static constexpr int THREADS_PER_GROUP_0 = WARPS_PER_GROUP_0 * THREADS_PER_WARP_0; + +// mma_pipeline_op 1 types +static constexpr int BYTES_PER_BANK_1 = 16; +static constexpr int BITS_PER_ELEMENT_1 = 16; +static constexpr int ELEMS_PER_BANK_1 = BYTES_PER_BANK_1 * 8 / BITS_PER_ELEMENT_1; +static constexpr int BYTES_PER_ELEMENT_1 = BITS_PER_ELEMENT_1 / 8; +static constexpr int BYTES_PER_FP16_1 = 2; +static constexpr int ELEMS_PER_VECTOR_1 = BYTES_PER_REGISTER_0 / BYTES_PER_ELEMENT_1; +static constexpr int BITS_PER_ACC_1 = 32; +static constexpr int BYTES_PER_ACC_1 = BITS_PER_ACC_1 / 8; +static constexpr int TILE_M_1 = 64; +static constexpr int TILE_N_1 = 128; +static constexpr int TILE_K_1 = 128; +static constexpr int TILE_O_1 = 128; +static constexpr int TILE_V_1 = TILE_K_1; +static constexpr int TILE_DV_1 = 128; +static constexpr int WARP_TILE_M_1 = 16; +static constexpr int WARP_TILE_N_1 = 16; +static constexpr int WARP_TILE_K_1 = 32 / BYTES_PER_ELEMENT_1; +static constexpr int WARPS_M_1 = 64 / 16; +static constexpr int WARPS_N_1 = 16 / 16; +static constexpr int WARPS_K_1 = 16 / 16; +static constexpr int STAGES_1 = 1; +static constexpr int GROUPS_M_1 = 1; +static constexpr int GMMA_TILE_M_1 = WARPS_M_1 * WARP_TILE_M_1; +static constexpr int GMMA_TILE_K_1 = WARPS_K_1 * WARP_TILE_K_1; +static constexpr int WARP_TILES_M_1 = TILE_M_1 / WARP_TILE_M_1; +static constexpr int WARP_TILES_N_1 = TILE_N_1 / WARP_TILE_N_1; +static constexpr int WARP_TILES_K_1 = TILE_K_1 / WARP_TILE_K_1; +static constexpr int WARP_REGS_1 = 8; +static constexpr int REGS_M_1 = WARP_TILES_M_1 / WARPS_M_1; +static constexpr int REGS_N_1 = WARP_TILES_N_1 / WARPS_N_1; +static constexpr int REGS_K_1 = WARP_TILES_K_1 / WARPS_K_1; +static constexpr int REGS_K_FOR_DP_1 = TILE_V_1 / 16; + +static constexpr int REGS_dQ_1 = TILE_O_1 / 16; +static constexpr int REGS_O_1 = 8; + +static constexpr int CGA_M_1 = 1; +static constexpr int CGA_N_1 = 1; +static constexpr int CTA_MMA_1 = 1; +static constexpr int M_TILES_PER_OUTPUT_TILE_1 = 4; +static constexpr int N_TILES_PER_OUTPUT_TILE_1 = 1; + +static constexpr int CTA_TILE_M_1 = TILE_M_1 * GROUPS_M_1; +static constexpr int CTA_TILE_N_1 = 128; +static constexpr int CTA_TILE_K_1 = 128; +static constexpr int BUFFERS_Q_1 = 2; +static constexpr int BUFFERS_K_1 = 2; +static constexpr int BUFFERS_V_1 = 2; +static constexpr int BUFFERS_O_1 = 1; +static constexpr int BUFFERS_D_1 = 1; +static constexpr int SMEM_Q_1 = TILE_M_1 * TILE_K_1 * BYTES_PER_ELEMENT_1 * 2; +static constexpr int SMEM_K_1 = TILE_N_1 * TILE_K_1 * BYTES_PER_ELEMENT_1; +static constexpr int SMEM_V_1 = TILE_N_1 * TILE_K_1 * BYTES_PER_ELEMENT_1; +static constexpr int SMEM_PAGE_TABLE_1 = FORT_ROUND_UP(TILE_N_1 * 2 * 2 * sizeof(int), 1024); +static constexpr int SMEM_OFFSET_Q_1 = 1024 + (false ? 0 : SMEM_PAGE_TABLE_1); +static constexpr int SMEM_OFFSET_K_1 = SMEM_OFFSET_Q_1 + SMEM_Q_1 * BUFFERS_Q_1; +static constexpr int SMEM_OFFSET_V_1 = SMEM_OFFSET_K_1 + SMEM_K_1 * BUFFERS_K_1; +static constexpr int SMEM_OFFSET_O_1 = SMEM_OFFSET_V_1 + SMEM_V_1 * BUFFERS_V_1; +static constexpr int SMEM_OFFSET_D_1 = SMEM_OFFSET_O_1; +static constexpr int SMEM_OFFSET_BAND_BIAS_1 = SMEM_OFFSET_O_1; + +static constexpr int THREADS_ON_TILE_BARRIER_1 = 256+32+0; +static constexpr int TILE_ID_SYNC_2_BARRIER_1 = 2; +static constexpr int TILE_ID_SYNC_3_BARRIER_1 = 3; +static constexpr int SOFTMAX_1_BARRIER_1 = 4; +static constexpr int SOFTMAX_2_BARRIER_1 = 5; +static constexpr int MATH_WORKGROUP_1 = 6; +static constexpr int TMA_SYNC_BARRIER_1 = 7; + +static constexpr uint32_t BYTES_PER_GMMA_K_1 = GMMA_TILE_K_1 * BYTES_PER_ELEMENT_1; +static constexpr uint32_t BYTES_PER_GMMA_K_NO_4LSB_1 = BYTES_PER_GMMA_K_1 >> 4; +static constexpr uint32_t BYTES_PER_GMMA_K_TRANS_1 = GMMA_TILE_K_1 * 128; +static constexpr uint32_t BYTES_PER_GMMA_K_NO_4LSB_TRANS_1 = BYTES_PER_GMMA_K_TRANS_1 >> 4; + +static constexpr uint32_t GMMA_DESCRIPTOR_SWIZZLE_MODE_BITS_1 = 2; +static constexpr uint32_t GMMA_DESCRIPTOR_SWIZZLE_MODE_SHIFT_1 = 62; // bits 63-62 +static constexpr uint64_t GMMA_DESCRIPTOR_SWIZZLE_MODE_VALUE_Q_1 = 1; // SWIZZLE_NONE=0, SWIZZLE_128B=1, SWIZZLE_64B=2, SWIZZLE_32B=3 +static constexpr uint64_t GMMA_DESCRIPTOR_SWIZZLE_MODE_IN_BIT_LOCATION_Q_1 = (GMMA_DESCRIPTOR_SWIZZLE_MODE_VALUE_Q_1 & +((1u << GMMA_DESCRIPTOR_SWIZZLE_MODE_BITS_1) - 1)) +<< GMMA_DESCRIPTOR_SWIZZLE_MODE_SHIFT_1; +static constexpr uint64_t GMMA_DESCRIPTOR_SWIZZLE_MODE_VALUE_K_1 = 1; // SWIZZLE_NONE=0, SWIZZLE_128B=1, SWIZZLE_64B=2, SWIZZLE_32B=3 +static constexpr uint64_t GMMA_DESCRIPTOR_SWIZZLE_MODE_IN_BIT_LOCATION_K_1 = (GMMA_DESCRIPTOR_SWIZZLE_MODE_VALUE_K_1 & +((1u << GMMA_DESCRIPTOR_SWIZZLE_MODE_BITS_1) - 1)) +<< GMMA_DESCRIPTOR_SWIZZLE_MODE_SHIFT_1; +static constexpr uint64_t GMMA_DESCRIPTOR_SWIZZLE_MODE_VALUE_V_1 = 1; // SWIZZLE_NONE=0, SWIZZLE_128B=1, SWIZZLE_64B=2, SWIZZLE_32B=3 +static constexpr uint64_t GMMA_DESCRIPTOR_SWIZZLE_MODE_IN_BIT_LOCATION_V_1 = (GMMA_DESCRIPTOR_SWIZZLE_MODE_VALUE_V_1 & +((1u << GMMA_DESCRIPTOR_SWIZZLE_MODE_BITS_1) - 1)) +<< GMMA_DESCRIPTOR_SWIZZLE_MODE_SHIFT_1; + +static constexpr uint32_t GMMA_DESCRIPTOR_STRIDE_BYTE_OFFSET_SHIFT_1 = 32; // bits 45-32 +static constexpr uint64_t BYTES_PER_LEADING_DIM_1 = 128; +static constexpr uint64_t STRIDE_BYTE_OFFSET_Q_1 = (BYTES_PER_LEADING_DIM_1 * 8) / 16; // SWIZZLE_NONE=0, SWIZZLE_128B=8, SWIZZLE_64B=4, SWIZZLE_32B=2 +static constexpr uint64_t GMMA_DESCRIPTOR_STRIDE_BYTE_OFFSET_IN_BIT_LOCATION_Q_1 = STRIDE_BYTE_OFFSET_Q_1 << GMMA_DESCRIPTOR_STRIDE_BYTE_OFFSET_SHIFT_1; +static constexpr uint64_t STRIDE_BYTE_OFFSET_K_1 = (BYTES_PER_LEADING_DIM_1 * 8) / 16; // SWIZZLE_NONE=0, SWIZZLE_128B=8, SWIZZLE_64B=4, SWIZZLE_32B=2 +static constexpr uint64_t GMMA_DESCRIPTOR_STRIDE_BYTE_OFFSET_IN_BIT_LOCATION_K_1 = STRIDE_BYTE_OFFSET_K_1 << GMMA_DESCRIPTOR_STRIDE_BYTE_OFFSET_SHIFT_1; +static constexpr uint64_t STRIDE_BYTE_OFFSET_V_1 = (BYTES_PER_LEADING_DIM_1 * 8) / 16; // SWIZZLE_NONE=0, SWIZZLE_128B=8, SWIZZLE_64B=4, SWIZZLE_32B=2 +static constexpr uint64_t GMMA_DESCRIPTOR_STRIDE_BYTE_OFFSET_IN_BIT_LOCATION_V_1 = STRIDE_BYTE_OFFSET_V_1 << GMMA_DESCRIPTOR_STRIDE_BYTE_OFFSET_SHIFT_1; + +static constexpr uint32_t GMMA_DESCRIPTOR_LD_BYTE_OFFSET_SHIFT_1 = 16; // bits 29-16 +static constexpr uint32_t GMMA_DESCRIPTOR_LD_BYTE_OFFSET_VALUE_Q_1 = 128 * 64 / 16; // Not used??? +static constexpr uint64_t GMMA_DESCRIPTOR_LD_BYTE_OFFSET_IN_BIT_LOCATION_Q_1 = GMMA_DESCRIPTOR_LD_BYTE_OFFSET_VALUE_Q_1 +<< GMMA_DESCRIPTOR_LD_BYTE_OFFSET_SHIFT_1; +static constexpr uint32_t GMMA_DESCRIPTOR_LD_BYTE_OFFSET_VALUE_K_1 = 128 * 32 / 16; // Not used??? +static constexpr uint64_t GMMA_DESCRIPTOR_LD_BYTE_OFFSET_IN_BIT_LOCATION_K_1 = GMMA_DESCRIPTOR_LD_BYTE_OFFSET_VALUE_K_1 +<< GMMA_DESCRIPTOR_LD_BYTE_OFFSET_SHIFT_1; +static constexpr uint32_t GMMA_DESCRIPTOR_LD_BYTE_OFFSET_VALUE_V_1 = 128 * TILE_N_1 / 16; +static constexpr uint64_t GMMA_DESCRIPTOR_LD_BYTE_OFFSET_IN_BIT_LOCATION_V_1 = GMMA_DESCRIPTOR_LD_BYTE_OFFSET_VALUE_V_1 +<< GMMA_DESCRIPTOR_LD_BYTE_OFFSET_SHIFT_1; + +inline __device__ constexpr uint64_t create_gmma_desc_q_1() { + return ((uint64_t)0 | GMMA_DESCRIPTOR_SWIZZLE_MODE_IN_BIT_LOCATION_Q_1) + | GMMA_DESCRIPTOR_STRIDE_BYTE_OFFSET_IN_BIT_LOCATION_Q_1 + | GMMA_DESCRIPTOR_LD_BYTE_OFFSET_IN_BIT_LOCATION_Q_1; +} + +inline __device__ constexpr uint64_t create_gmma_desc_k_1() { + return ((uint64_t)0 | GMMA_DESCRIPTOR_SWIZZLE_MODE_IN_BIT_LOCATION_K_1) + | GMMA_DESCRIPTOR_STRIDE_BYTE_OFFSET_IN_BIT_LOCATION_K_1 + | GMMA_DESCRIPTOR_LD_BYTE_OFFSET_IN_BIT_LOCATION_K_1; +} + +// === Section: GMMA Descriptors & Mask Helpers === +)KERNEL" + R"KERNEL( +// === Section: GMMA Descriptors & Mask Helpers === +inline __device__ constexpr uint64_t create_gmma_desc_v_1() { + return ((uint64_t)0 | GMMA_DESCRIPTOR_SWIZZLE_MODE_IN_BIT_LOCATION_V_1) + | GMMA_DESCRIPTOR_STRIDE_BYTE_OFFSET_IN_BIT_LOCATION_V_1 + | GMMA_DESCRIPTOR_LD_BYTE_OFFSET_IN_BIT_LOCATION_V_1; +} + +class Gmma_descriptor { + public: + inline __device__ Gmma_descriptor(uint64_t desc_) { + desc = desc_; + } + template + inline __device__ void set_smem(uint32_t smem) { + desc |= (static_cast(smem & 0xFFFFFF) >> 4); + + max_desc = desc + (BYTES_PER_BUFFER >> 4) * (BUFFER_COUNT - 1); + } + template + inline __device__ void increment_smem_buffer() { + int2 &tmp = reinterpret_cast(desc); + tmp.x += (desc >= max_desc) ? -(BYTES_PER_BUFFER >> 4) * (BUFFER_COUNT - 1) + : (BYTES_PER_BUFFER >> 4); + } + + uint64_t desc; + uint64_t max_desc; +}; + +//global_load_shared_store_op 4 types +static constexpr int BITS_PER_ELEMENT_4 = 16; +static constexpr int BYTES_PER_ELEMENT_4 = BITS_PER_ELEMENT_4 / 8; +static constexpr int BYTES_PER_ACCESS_4 = 16; +static constexpr int TILE_M_4 = 64; +static constexpr int TILE_N_4 = 128; +static constexpr int BYTES_PER_SMEM_4 = TILE_M_4 * TILE_N_4 * BYTES_PER_ELEMENT_4; + +// shared_load_op 5 types +static constexpr int BITS_PER_ELEMENT_5 = 16; +static constexpr int BYTES_PER_ELEMENT_5 = 2; +static constexpr int BYTES_PER_QUAD_5 = 16; +static constexpr int WARP_TILE_M_5 = 16; +static constexpr int WARP_TILE_N_5 = 16; +static constexpr int TILE_M_5 = 64; +static constexpr int TILE_N_5 = 128; +static constexpr int BYTES_PER_SMEM_5 = TILE_M_5 * TILE_N_5 * BYTES_PER_ELEMENT_5; +static constexpr int WARPS_M_5 = 4; +static constexpr int WARPS_N_5 = 1; +static constexpr int WARP_TILES_M_5 = TILE_M_5 / WARP_TILE_M_5; +static constexpr int WARP_TILES_N_5 = TILE_N_5 / WARP_TILE_N_5; +static constexpr int THREADS_PER_WARP_TILE_M_5 = 8; +static constexpr int THREADS_PER_WARP_TILE_N_5 = 4; +static constexpr int REGS_M_5 = WARP_TILES_M_5 / WARPS_M_5; +static constexpr int REGS_N_5 = WARP_TILES_N_5 / WARPS_N_5; +static constexpr int BYTES_PER_LD_5 = 128; +static constexpr int SWIZZLE_SCALE_5 = FORT_MIN(BYTES_PER_LD_5 / 16, WARPS_PER_GROUP_0 * 4); + +// shared_load_op 6 types +static constexpr int BITS_PER_ELEMENT_6 = 16; +static constexpr int BYTES_PER_ELEMENT_6 = 2; +static constexpr int BYTES_PER_QUAD_6 = 16; +static constexpr int WARP_TILE_M_6 = 16; +static constexpr int WARP_TILE_N_6 = 16; +static constexpr int TILE_M_6 = 128; +static constexpr int TILE_N_6 = 128; + +static constexpr int BYTES_PER_SMEM_6 = TILE_M_6 * TILE_N_6 * BYTES_PER_ELEMENT_6; +static constexpr int WARPS_M_6 = 1; +static constexpr int WARPS_N_6 = 1; +static constexpr int WARP_TILES_M_6 = TILE_M_6 / WARP_TILE_M_6; +static constexpr int WARP_TILES_N_6 = TILE_N_6 / WARP_TILE_N_6; +static constexpr int THREADS_PER_WARP_TILE_M_6 = 8; +static constexpr int THREADS_PER_WARP_TILE_N_6 = 4; +static constexpr int REGS_M_6 = WARP_TILES_M_6 / WARPS_M_6; +static constexpr int REGS_N_6 = WARP_TILES_N_6 / WARPS_N_6; +static constexpr int BYTES_PER_LD_6 = 128; +static constexpr int SWIZZLE_SCALE_6 = FORT_MIN(BYTES_PER_LD_6 / 16, WARPS_PER_GROUP_0 * 4); + +//global_load_shared_store_op 7 types +static constexpr int BITS_PER_ELEMENT_7 = 16; +static constexpr int BYTES_PER_ELEMENT_7 = BITS_PER_ELEMENT_7 / 8; +static constexpr int BYTES_PER_ACCESS_7 = 16; +static constexpr int TILE_M_7 = 128; +static constexpr int TILE_N_7 = 128; +static constexpr int BYTES_PER_SMEM_7 = TILE_M_7 * TILE_N_7 * BYTES_PER_ELEMENT_7; + +//mma_op 8 types +static constexpr int BYTES_PER_ELEMENT_8 = 2; +static constexpr int BYTES_PER_ACC_8 = 4; +static constexpr int WARP_TILE_M_8 = 16; +static constexpr int WARP_TILE_N_8 = 16; +static constexpr int WARP_TILE_K_8 = 16; +static constexpr int TILE_M_8 = 64; +static constexpr int TILE_N_8 = 128; +static constexpr int TILE_K_8 = 128; +static constexpr int WARPS_M_8 = 4; +static constexpr int WARPS_N_8 = 1; +static constexpr int WARP_TILES_M_8 = TILE_M_8 / WARP_TILE_M_8; +static constexpr int WARP_TILES_N_8 = TILE_N_8 / WARP_TILE_N_8; +static constexpr int MMA_STEPS_K_8 = TILE_K_8 / WARP_TILE_K_8; +static constexpr int WARP_REGS_8 = 8; +static constexpr int REGS_M_8 = WARP_TILES_M_8 / WARPS_M_8; +static constexpr int REGS_N_8 = WARP_TILES_N_8 / WARPS_N_8; + +//pointwise_calc_op 9 types +static constexpr int WARP_TILE_M_9 = 16; +static constexpr int WARP_TILE_N_9 = 16; +static constexpr int TILE_M_9 = 64; +static constexpr int TILE_N_9 = 128; +static constexpr int WARPS_M_9 = 4; +static constexpr int WARPS_N_9 = 1; +static constexpr int WARP_TILES_M_9 = TILE_M_9 / WARP_TILE_M_9; +static constexpr int WARP_TILES_N_9 = TILE_N_9 / WARP_TILE_N_9; +static constexpr int WARP_REGS_9 = 8; +static constexpr int REGS_M_9 = WARP_TILES_M_9 / WARPS_M_9; +static constexpr int REGS_N_9 = WARP_TILES_N_9 / WARPS_N_9; + +//global_load_op 10 types +static constexpr int BITS_PER_ELEMENT_10 = 32; +static constexpr int BYTES_PER_ELEMENT_10 = 4; +static constexpr int REGISTERS_PER_VECTOR_10 = 1; +static constexpr int REGISTERS_PER_ACCESS_10 = 1; + +//mha_mask_op 11 types +static constexpr int WARP_TILE_M_11 = 16; +static constexpr int WARP_TILE_N_11 = 16; +static constexpr int TILE_M_11 = 64; +static constexpr int TILE_N_11 = 128; +static constexpr int WARPS_M_11 = 4; +static constexpr int WARPS_N_11 = 1; +static constexpr int WARP_TILES_M_11 = TILE_M_11 / WARP_TILE_M_11; +static constexpr int WARP_TILES_N_11 = TILE_N_11 / WARP_TILE_N_11; +static constexpr int WARP_REGS_11 = 8; +static constexpr int REGS_M_11 = WARP_TILES_M_11 / WARPS_M_11; +static constexpr int REGS_N_11 = WARP_TILES_N_11 / WARPS_N_11; +inline __device__ bool compute_diagonal_band_mask_11(const int row, const int col, const int actual_seqlen_kv, const int actual_seqlen_q, const int shift_right_bound, const int left_bound) { + constexpr bool is_bottom_right_alignment = false; + constexpr bool is_shift_right_bound = false; + constexpr bool is_right_bound = true; + constexpr bool is_left_bound = false; + constexpr bool need_oob_check = false; + + const int diag = is_bottom_right_alignment ? row + (actual_seqlen_kv - actual_seqlen_q) : row; + const int shifted_right_bound = is_shift_right_bound ? diag + shift_right_bound : diag; + const bool right_bound_mask = is_right_bound ? col <= shifted_right_bound : true; + const bool left_bound_mask = is_left_bound ? col + left_bound > diag : true; + const bool oob_check = need_oob_check ? (row < actual_seqlen_q && col < actual_seqlen_kv) : true; + return right_bound_mask && left_bound_mask && oob_check; +} + +//softmax_op 12 types +static constexpr int BYTES_PER_ELEMENT_12 = 2; +static constexpr int WARP_TILE_M_12 = 16; +static constexpr int WARP_TILE_N_12 = 16; +static constexpr int TILE_M_12 = 64; +static constexpr int TILE_N_12 = 128; +static constexpr int WARPS_M_12 = 4; +static constexpr int WARPS_N_12 = 1; +static constexpr int WARP_TILES_M_12 = TILE_M_12 / WARP_TILE_M_12; + +static constexpr int WARP_TILES_N_12 = TILE_N_12 / WARP_TILE_N_12; +static constexpr int THREADS_PER_WARP_TILE_M_12 = 8; +static constexpr int THREADS_PER_WARP_TILE_N_12 = 4; +static constexpr int WARP_REGS_12 = 8; +static constexpr int REGS_M_12 = WARP_TILES_M_12 / WARPS_M_12; +static constexpr int REGS_N_12 = WARP_TILES_N_12 / WARPS_N_12; +static constexpr int ROWS_PER_THREAD_12 = TILE_M_12 / WARPS_M_12 / THREADS_PER_WARP_TILE_M_12; + +// global_store_op 13 types +static constexpr int BYTES_PER_ELEMENT_13 = 4; +static constexpr int WARP_TILE_M_13 = 16; +static constexpr int WARP_TILE_N_13 = 16; +static constexpr int TILE_M_13 = 64; +static constexpr int TILE_N_13 = 128; +static constexpr int WARPS_M_13 = 4; +static constexpr int WARPS_N_13 = 1; +static constexpr int WARP_TILES_M_13 = TILE_M_13 / WARP_TILE_M_13; +static constexpr int WARP_TILES_N_13 = TILE_N_13 / WARP_TILE_N_13; +static constexpr int THREADS_PER_WARP_TILE_M_13 = 8; +static constexpr int THREADS_PER_WARP_TILE_N_13 = 4; +static constexpr int ROWS_PER_THREAD_13 = TILE_M_13 / WARPS_M_13 / THREADS_PER_WARP_TILE_M_13; + +// global_store_op 14 types +static constexpr int BYTES_PER_ELEMENT_14 = 4; +static constexpr int WARP_TILE_M_14 = 16; +static constexpr int WARP_TILE_N_14 = 16; +static constexpr int TILE_M_14 = 64; +static constexpr int TILE_N_14 = 128; +static constexpr int WARPS_M_14 = 4; +static constexpr int WARPS_N_14 = 1; +static constexpr int WARP_TILES_M_14 = TILE_M_14 / WARP_TILE_M_14; +static constexpr int WARP_TILES_N_14 = TILE_N_14 / WARP_TILE_N_14; +static constexpr int THREADS_PER_WARP_TILE_M_14 = 8; +static constexpr int THREADS_PER_WARP_TILE_N_14 = 4; +static constexpr int ROWS_PER_THREAD_14 = TILE_M_14 / WARPS_M_14 / THREADS_PER_WARP_TILE_M_14; + +// shared_load_op 15 types +static constexpr int BITS_PER_ELEMENT_15 = 16; +static constexpr int BYTES_PER_ELEMENT_15 = 2; +static constexpr int BYTES_PER_QUAD_15 = 16; +static constexpr int WARP_TILE_M_15 = 16; +static constexpr int WARP_TILE_N_15 = 16; +static constexpr int TILE_M_15 = 128; +static constexpr int TILE_N_15 = 128; +static constexpr int BYTES_PER_SMEM_15 = TILE_M_15 * TILE_N_15 * BYTES_PER_ELEMENT_15; +static constexpr int WARPS_M_15 = 1; +static constexpr int WARPS_N_15 = 1; +static constexpr int WARP_TILES_M_15 = TILE_M_15 / WARP_TILE_M_15; +static constexpr int WARP_TILES_N_15 = TILE_N_15 / WARP_TILE_N_15; +static constexpr int THREADS_PER_WARP_TILE_M_15 = 8; +static constexpr int THREADS_PER_WARP_TILE_N_15 = 4; +static constexpr int REGS_M_15 = WARP_TILES_M_15 / WARPS_M_15; +static constexpr int REGS_N_15 = WARP_TILES_N_15 / WARPS_N_15; +static constexpr int BYTES_PER_LD_15 = 128; +static constexpr int SWIZZLE_SCALE_15 = FORT_MIN(BYTES_PER_LD_15 / 16, WARPS_PER_GROUP_0 * 4); + +//global_load_shared_store_op 16 types +static constexpr int BITS_PER_ELEMENT_16 = 16; +static constexpr int BYTES_PER_ELEMENT_16 = BITS_PER_ELEMENT_16 / 8; +static constexpr int BYTES_PER_ACCESS_16 = 16; +static constexpr int TILE_M_16 = 128; +static constexpr int TILE_N_16 = 128; +static constexpr int BYTES_PER_SMEM_16 = TILE_M_16 * TILE_N_16 * BYTES_PER_ELEMENT_16; + +//mma_op 17 types +static constexpr int BYTES_PER_ELEMENT_17 = 2; +static constexpr int BYTES_PER_ACC_17 = 4; +static constexpr int WARP_TILE_M_17 = 16; +static constexpr int WARP_TILE_N_17 = 16; +static constexpr int WARP_TILE_K_17 = 16; +static constexpr int TILE_M_17 = 64; +static constexpr int TILE_N_17 = 128; +static constexpr int TILE_K_17 = 128; +static constexpr int WARPS_M_17 = 4; +static constexpr int WARPS_N_17 = 1; +static constexpr int WARP_TILES_M_17 = TILE_M_17 / WARP_TILE_M_17; +static constexpr int WARP_TILES_N_17 = TILE_N_17 / WARP_TILE_N_17; +static constexpr int MMA_STEPS_K_17 = TILE_K_17 / WARP_TILE_K_17; +static constexpr int WARP_REGS_17 = 8; +static constexpr int REGS_M_17 = WARP_TILES_M_17 / WARPS_M_17; +static constexpr int REGS_N_17 = WARP_TILES_N_17 / WARPS_N_17; + +// global_store_op 20 types + +// mma_loop_op 2 types +inline __device__ int2 compute_kv_loop_bounds(const int row_coord, const int ROW_TILE_SIZE, const int COL_TILE_SIZE, const int actual_seqlen_kv, const int actual_seqlen_q, const int shift_right_bound, const int left_bound) +{ + constexpr bool is_right_bound = true; + constexpr bool is_right_bound_bottom_right_alignment = false; + constexpr bool is_shift_right_bound = false; + constexpr bool is_left_bound = false; + constexpr bool is_left_bound_bottom_right_alignment = false; + + const int right_bound_diagonal = is_right_bound_bottom_right_alignment ? row_coord + (actual_seqlen_kv - actual_seqlen_q) : row_coord; + const int shifted_right_bound_diagonal = is_shift_right_bound ? right_bound_diagonal + shift_right_bound : right_bound_diagonal; + const int left_bound_diagonal = is_left_bound_bottom_right_alignment ? row_coord + (actual_seqlen_kv - actual_seqlen_q) : row_coord; + + const int kv_loop_left_bound = is_left_bound ? FORT_MAX(0, (left_bound_diagonal - left_bound) / COL_TILE_SIZE) : 0; + const int kv_loop_right_bound = is_right_bound ? FORT_MIN(FORT_DIV_UP(shifted_right_bound_diagonal + ROW_TILE_SIZE, COL_TILE_SIZE), FORT_DIV_UP(actual_seqlen_kv, COL_TILE_SIZE)) : FORT_DIV_UP(actual_seqlen_kv, COL_TILE_SIZE); + return make_int2(kv_loop_left_bound, kv_loop_right_bound); +} + +// output_loop_op 3 types +static constexpr int BYTES_PER_BANK_3 = 16; +static constexpr int LDS_TILE_M_3 = 16; +static constexpr int LDS_TILE_N_3 = 32; +static constexpr int ACC_CORE_MATRIX_ROWS_3 = 8; // NOTE: fixed due to GMMA design +static constexpr int ACC_CORE_MATRIX_COLS_3 = 8; // NOTE: fixed due to GMMA design +static constexpr int ELEMENTS_PER_VECTOR_3 = 8; +static constexpr int LDS_PER_TILE_3 = ELEMENTS_PER_VECTOR_3 / 2; +static constexpr int LDS_TILES_N_3 = FORT_MAX(CTA_TILE_N_1 / LDS_TILE_N_3, 1); +static constexpr int LDS_TILES_M_3 = FORT_MAX(1, (CTA_TILE_M_1 / GROUPS_M_1) / (LDS_TILE_M_3 * WARPS_PER_GROUP_0)); +static constexpr int VECTORS_PER_LDS_TILES_3 = LDS_TILE_M_3 / ACC_CORE_MATRIX_ROWS_3; +static constexpr int STS_PER_OUTPUT_TILE_N_3 = CTA_TILE_N_1 / ACC_CORE_MATRIX_COLS_3; +static constexpr int STSM_X4_PER_OUTPUT_TILE_N_3 = CTA_TILE_N_1 / ACC_CORE_MATRIX_COLS_3 / 2; +static constexpr int BANKS_PER_PAD_LINE_3 = BYTES_PER_ACC_1 == 4 ? 1 : 2; + +static constexpr int PADDING_BYTES_3 = BANKS_PER_PAD_LINE_3 * BYTES_PER_BANK_3 * LDS_PER_TILE_3; +)KERNEL" + R"KERNEL( +static constexpr int BYTES_PER_STS_PER_WARP_3= 16 * THREADS_PER_WARP_0; // 16 bytes due to STS_128 +static constexpr int BYTES_PER_LDS_TILE_3 = LDS_TILE_N_3 * LDS_TILE_M_3 * BYTES_PER_ACC_1 + PADDING_BYTES_3; +static constexpr int BYTES_PER_WARP_3 = LDS_TILES_N_3 * BYTES_PER_LDS_TILE_3; + +static constexpr int ELEMS_PER_STS_BLOCK_3 = 16; + +static constexpr int STG_THREADS_PER_TILE_N_3 = LDS_TILE_N_3 / ELEMENTS_PER_VECTOR_3; +static constexpr int EPILOGUE_SMEM_SIZE_PER_XMMA_M_3 = BYTES_PER_WARP_3 * WARPS_PER_GROUP_0; + +// shared_store_op 18 types + +// shared_load_op 19 types +static constexpr int BITS_PER_ELEMENT_19 = 32 * 1; +static constexpr int BYTES_PER_ELEMENT_19 = BITS_PER_ELEMENT_19 / 8; +static constexpr int BITS_PER_VECTOR_19 = BITS_PER_ELEMENT_19 * ELEMENTS_PER_VECTOR_3; +static constexpr int BYTES_PER_VECTOR_19 = BITS_PER_VECTOR_19 / 8; +static constexpr int REGISTERS_PER_VECTOR_19 = BITS_PER_VECTOR_19 / BITS_PER_REGISTER_0; + +//receive_op 0 code + +// === Section: Kernel Function === +)KERNEL" + R"KERNEL( +// === Section: Kernel Function === +extern "C" __global__ __launch_bounds__(384, 1) +void cudnn_generated_oss_sdpa_sm90_flash_fprop_wgmma_f16_knob_7_64x128x128_4x1x1_cga1x1x1_kernel0_0(const AttentionDescriptor_t attnDesc +, int num_tiles_1 +, const FastDivisor_t tiles_hr_div_1 +, const FastDivisor_t tiles_r_div_1 +, int32_t* tile_id_counter +, __grid_constant__ const CUtensorMap tma_Q +, __grid_constant__ const CUtensorMap tma_K +, float attn_scale +, float neg_infinity +, void* d_max +, fort::tensor_descriptor desc_max +, void* d_sum_exp +, fort::tensor_descriptor desc_sum_exp +, __grid_constant__ const CUtensorMap tma_V +, __grid_constant__ const CUtensorMap tma_O +) { + extern __shared__ char smem_[]; + uint32_t smem_0 = get_smem_pointer(smem_); + + //receive_op 0 decls + + // mma_pipeline_op 1 decls + const uint32_t tid = threadIdx.x % THREADS_PER_GROUP_0; + const uint32_t gid = threadIdx.x / THREADS_PER_GROUP_0; + const uint32_t wid = threadIdx.x / 32; + const uint32_t tiw = threadIdx.x % 32; + + const uint32_t elect_one = fort::elect_one_sync(); + + uint32_t smem_1 = smem_0; + uint32_t smem_bar_qkv_1 = smem_1; + uint32_t smem_page = smem_1 + 1024; + uint32_t smem_Q = smem_1 + SMEM_OFFSET_Q_1; + uint32_t smem_K = smem_1 + SMEM_OFFSET_K_1; + uint32_t smem_V = smem_1 + SMEM_OFFSET_V_1; + uint32_t smem_O = smem_1 + SMEM_OFFSET_O_1; + uint32_t smem_d_1 = smem_1 + SMEM_OFFSET_D_1; + uint32_t smem_band_bias_1 = smem_1 + SMEM_OFFSET_BAND_BIAS_1; + + uint32_t smem_bar_qkv_tma_1 = smem_bar_qkv_1; + uint32_t smem_bar_qkv_mma_1 = smem_bar_qkv_1 + 1 * 8 * (BUFFERS_Q_1 + BUFFERS_K_1 + BUFFERS_V_1 + BUFFERS_D_1 + 0); + uint32_t smem_tile_id_1 = smem_bar_qkv_1 + 2 * 8 * (BUFFERS_Q_1 + BUFFERS_K_1 + BUFFERS_V_1 + BUFFERS_D_1 + 0); + uint32_t smem_bar_tile_id_tma_1 = smem_tile_id_1 + 16; + uint32_t smem_bar_bias_tma_1 = smem_bar_tile_id_tma_1 + 8; + + uint32_t smem_bar_bias_mma_1 = smem_bar_bias_tma_1 + 8 * 2; + + if (elect_one) { + for (int i = wid; i < (BUFFERS_Q_1 + BUFFERS_K_1 + BUFFERS_V_1 + BUFFERS_D_1 + 0); i+=12) { + smem_bar_init(smem_bar_qkv_tma_1 + i * 8, 1); + smem_bar_init(smem_bar_qkv_mma_1 + i * 8, 2); + } + } + __syncthreads(); + + // mma_pipeline_op 1 code + const int actual_num_tiles_1 = num_tiles_1; + if (gid == 0) { + reg_dealloc<40>(); + const uint32_t local_wid = wid % 4; + + uint32_t tile_id; + uint32_t batch_coord_1 = 0, head_coord_1 = 0, q_row_coord = 0, rows_temp; + uint32_t head_coord_from_grid_1, head_coord_k_1, head_coord_v_1, head_coord_q_offset_1; + int actual_seqlen_kv_1 = attnDesc.s_kv, actual_seqlen_q_1 = attnDesc.s_q, oob_for_stats_1 = attnDesc.s_q; + + if (local_wid == 0) { + uint32_t smem_bar_q_tma = smem_bar_qkv_tma_1; + uint32_t smem_bar_k_tma = smem_bar_q_tma + 8 * BUFFERS_Q_1; + uint32_t smem_bar_v_tma = smem_bar_k_tma + 8 * BUFFERS_K_1; + + uint32_t smem_bar_q_mma = smem_bar_qkv_mma_1; + uint32_t smem_bar_k_mma = smem_bar_q_mma + 8 * BUFFERS_Q_1; + uint32_t smem_bar_v_mma = smem_bar_k_mma + 8 * BUFFERS_K_1; + + uint32_t buffer_id_q = 0; + uint32_t buffer_id_kv = 0; + uint32_t cnt_q = 0; + uint32_t cnt_kv = 0; + + uint32_t sts_page_buffer_id = 0; + uint32_t lds_page_buffer_id = 0; + + int cumulative_tiles = 0; + if (tiw == 0) { + tile_id = atomicAdd(tile_id_counter, 1); + } + tile_id = __shfl_sync(0xffffffff, tile_id, 0); + + named_barrier_wait(TILE_ID_SYNC_2_BARRIER_1, THREADS_ON_TILE_BARRIER_1); + if (tiw == 0) { + sts_32(smem_tile_id_1, &tile_id); + } + named_barrier_arrive(TILE_ID_SYNC_3_BARRIER_1, THREADS_ON_TILE_BARRIER_1); + + // Persistent loop over output tiles +#pragma unroll 1 + while (tile_id < actual_num_tiles_1) { + fastDivMod(tiles_hr_div_1, tile_id, batch_coord_1, rows_temp); + fastDivMod(tiles_r_div_1, rows_temp, head_coord_from_grid_1, q_row_coord); + q_row_coord *= TILE_M_1 * 2; + head_coord_1 = head_coord_from_grid_1; + head_coord_k_1 = head_coord_1 / attnDesc.q_heads_per_k; + head_coord_v_1 = head_coord_1 / attnDesc.q_heads_per_v; + + const int left_bound = 0; + const int shift_right_bound = 0; + + int2 kv_loop_bounds = compute_kv_loop_bounds(q_row_coord, TILE_M_1 * 2, TILE_N_1, actual_seqlen_kv_1, actual_seqlen_q_1, shift_right_bound, left_bound); + int kv_loop_left_bound = kv_loop_bounds.x; + int kv_loop_right_bound = kv_loop_bounds.y; + + //global_load_shared_store_op 4 decls + + //global_load_shared_store_op 7 decls + + //global_load_shared_store_op 16 decls + + uint32_t local_smem_q = smem_Q + buffer_id_q * SMEM_Q_1; + uint32_t local_smem_bar_tma_q = smem_bar_q_tma + buffer_id_q * 8; + uint32_t local_smem_bar_mma_q = smem_bar_q_mma + buffer_id_q * 8; + + if (kv_loop_left_bound < kv_loop_right_bound) { + uint32_t smem_bar_phase_q = cnt_q < BUFFERS_Q_1 ? 1 : 0; + + cnt_q = cnt_q < (2 * BUFFERS_Q_1 - 1) ? (cnt_q + 1) : 0; + buffer_id_q = buffer_id_q < ( BUFFERS_Q_1 - 1) ? (buffer_id_q + 1) : 0; + + if (!smem_bar_peek(local_smem_bar_mma_q, smem_bar_phase_q)) { + smem_bar_wait(local_smem_bar_mma_q, smem_bar_phase_q); + } + + smem_bar_set_transaction_count(local_smem_bar_tma_q, SMEM_Q_1, elect_one); + { + const uint32_t row_coord = q_row_coord; + + //global_load_shared_store_op 4 code +#pragma unroll + for (int i = 0; i < TILE_K_1 * BYTES_PER_ELEMENT_1; i += 128) { + utmaldg_4d_tiled(&tma_Q, + local_smem_q + i * TILE_M_1 * 2, + local_smem_bar_tma_q, + i / BYTES_PER_ELEMENT_1, + row_coord, + head_coord_1 + 0, + batch_coord_1, + elect_one); + } + } + } + +#pragma unroll 1 + for (int kv_loop = kv_loop_left_bound; kv_loop < kv_loop_right_bound; kv_loop++) { + const int p_col_coord = kv_loop * TILE_N_1; + + uint32_t local_smem_k = smem_K + buffer_id_kv * SMEM_K_1; + uint32_t local_smem_bar_tma_k = smem_bar_k_tma + buffer_id_kv * 8; + uint32_t local_smem_bar_mma_k = smem_bar_k_mma + buffer_id_kv * 8; + + uint32_t local_smem_v = smem_V + buffer_id_kv * SMEM_V_1; + uint32_t local_smem_bar_tma_v = smem_bar_v_tma + buffer_id_kv * 8; + uint32_t local_smem_bar_mma_v = smem_bar_v_mma + buffer_id_kv * 8; + + uint32_t smem_bar_phase_kv = cnt_kv < BUFFERS_K_1 ? 1 : 0; + + cnt_kv = cnt_kv < (2 * BUFFERS_K_1 - 1) ? (cnt_kv + 1) : 0; + buffer_id_kv = buffer_id_kv < ( BUFFERS_K_1 - 1) ? (buffer_id_kv + 1) : 0; + + uint32_t kv_col_base = (kv_loop + 1) * TILE_N_1; + + if (!smem_bar_peek(local_smem_bar_mma_k, smem_bar_phase_kv)) { + smem_bar_wait(local_smem_bar_mma_k, smem_bar_phase_kv); + } + smem_bar_set_transaction_count(local_smem_bar_tma_k, SMEM_K_1, elect_one); + + //global_load_shared_store_op 7 code +#pragma unroll + for (int i = 0; i < TILE_K_1 * BYTES_PER_ELEMENT_1; i += 128) { + utmaldg_4d_tiled(&tma_K, + local_smem_k + i * TILE_N_1, + local_smem_bar_tma_k, + i / BYTES_PER_ELEMENT_1, + p_col_coord, + head_coord_k_1, + batch_coord_1, + elect_one); + } + if (!smem_bar_peek(local_smem_bar_mma_v, smem_bar_phase_kv)) { + smem_bar_wait(local_smem_bar_mma_v, smem_bar_phase_kv); + } + smem_bar_set_transaction_count(local_smem_bar_tma_v, SMEM_V_1, elect_one); + + //global_load_shared_store_op 16 code +#pragma unroll + for (int i = 0; i < TILE_K_1 * BYTES_PER_ELEMENT_1; i += 128 * CTA_MMA_1) { + utmaldg_4d_tiled(&tma_V, + local_smem_v + i * TILE_N_1, + local_smem_bar_tma_v, + 0 + i / BYTES_PER_ELEMENT_1, + p_col_coord, + head_coord_v_1, + batch_coord_1, + elect_one); + } + lds_page_buffer_id ^= 1; + } + if (kv_loop_left_bound < kv_loop_right_bound) { + lds_page_buffer_id ^= 1; + } + + if (tiw == 0) { + tile_id = atomicAdd(tile_id_counter, 1); + } + tile_id = __shfl_sync(0xffffffff, tile_id, 0); + + named_barrier_wait(TILE_ID_SYNC_2_BARRIER_1, THREADS_ON_TILE_BARRIER_1); + if (tiw == 0) { + sts_32(smem_tile_id_1, &tile_id); + } + named_barrier_arrive(TILE_ID_SYNC_3_BARRIER_1, THREADS_ON_TILE_BARRIER_1); + } + named_barrier_wait(TILE_ID_SYNC_2_BARRIER_1, THREADS_ON_TILE_BARRIER_1); + } + reg_dealloc<40>(); + } else { + reg_alloc<232>(); + named_barrier_arrive(TILE_ID_SYNC_2_BARRIER_1, THREADS_ON_TILE_BARRIER_1); + + const uint32_t local_wid = wid % 4; + const uint32_t local_gid = gid % 2; + + uint32_t tile_id; + uint32_t batch_coord_1, head_coord_1, q_row_coord, rows_temp; + uint32_t head_coord_from_grid_1, head_coord_k_1, head_coord_v_1, head_coord_q_offset_1; + int actual_seqlen_kv_1 = attnDesc.s_kv, actual_seqlen_q_1 = attnDesc.s_q, oob_for_stats_1 = attnDesc.s_q; + + uint32_t smem_bar_q_tma = smem_bar_qkv_tma_1; + uint32_t smem_bar_k_tma = smem_bar_q_tma + 8 * BUFFERS_Q_1; + uint32_t smem_bar_v_tma = smem_bar_k_tma + 8 * BUFFERS_K_1; + + uint32_t smem_bar_q_mma = smem_bar_qkv_mma_1; + uint32_t smem_bar_k_mma = smem_bar_q_mma + 8 * BUFFERS_Q_1; + uint32_t smem_bar_v_mma = smem_bar_k_mma + 8 * BUFFERS_K_1; + + uint32_t buffer_id_q = 0; + uint32_t buffer_id_kv = 0; + uint32_t cnt_q = 0; + uint32_t cnt_kv = 0; + + static constexpr float inv_ln2 = 1.4426950408889634074f; + static constexpr float ln2 = 0.6931471805599453094f; + + Gmma_descriptor gmma_desc_q(create_gmma_desc_q_1()); + Gmma_descriptor gmma_desc_k(create_gmma_desc_k_1()); + Gmma_descriptor gmma_desc_v(create_gmma_desc_v_1()); + + gmma_desc_q.set_smem(smem_Q + local_gid * TILE_M_1 * 128); + gmma_desc_k.set_smem(smem_K); + gmma_desc_v.set_smem(smem_V); + + named_barrier_wait(TILE_ID_SYNC_3_BARRIER_1, THREADS_ON_TILE_BARRIER_1); + if (tiw == 0) { + lds_32(&tile_id, smem_tile_id_1); + } + named_barrier_arrive(TILE_ID_SYNC_2_BARRIER_1, THREADS_ON_TILE_BARRIER_1); + tile_id = __shfl_sync(0xffffffff, tile_id, 0); + + // Persistent loop over output tiles +#pragma unroll 1 + while (tile_id < actual_num_tiles_1) { + fastDivMod(tiles_hr_div_1, tile_id, batch_coord_1, rows_temp); + fastDivMod(tiles_r_div_1, rows_temp, head_coord_from_grid_1, q_row_coord); + q_row_coord = q_row_coord * TILE_M_1 * 2; + head_coord_1 = head_coord_from_grid_1; + head_coord_k_1 = head_coord_1 / attnDesc.q_heads_per_k; + head_coord_v_1 = head_coord_1 / attnDesc.q_heads_per_v; + const int left_bound = 0; + const int shift_right_bound = 0; + + int2 kv_loop_bounds = compute_kv_loop_bounds(q_row_coord, TILE_M_1 * 2, TILE_N_1, actual_seqlen_kv_1, actual_seqlen_q_1, shift_right_bound, left_bound); + int kv_loop_left_bound = kv_loop_bounds.x; + int kv_loop_right_bound = kv_loop_bounds.y; + q_row_coord += local_gid * TILE_M_1; + + const int causal_mask_col = (tiw % 4) * 2; + const int causal_mask_row = q_row_coord + local_wid * 16 + (tiw / 4); + + // mma_loop_op 2 decls + + //global_load_shared_store_op 4 decls + + // shared_load_op 5 decls + + // shared_load_op 6 decls + + //global_load_shared_store_op 7 decls + + //mma_op 8 decls + r32 reg_8_0[REGS_M_8][REGS_N_8][WARP_REGS_8]; + + //pointwise_calc_op 9 decls + + //global_load_op 10 decls + r32 reg_10_0[1]; + reg_10_0[0] = reinterpret_cast(attn_scale); + const int oob_M_10 = 1; + const int oob_N_10 = 1; + + //mha_mask_op 11 decls + + //softmax_op 12 decls + r32 reg_12_0[REGS_M_12][REGS_N_12][4]; // output for second gemm + r32 reg_12_1[ROWS_PER_THREAD_12]; // acc_max + r32 reg_12_2[ROWS_PER_THREAD_12]; // p_max + r32 reg_12_3[ROWS_PER_THREAD_12]; // acc_sum + r32 reg_12_4[ROWS_PER_THREAD_12]; // p_sum + r32 reg_12_5[ROWS_PER_THREAD_12]; // softmax stats + float beta_12[ROWS_PER_THREAD_12]; + +#pragma unroll + for (int i = 0; i < ROWS_PER_THREAD_12; ++i) { + reinterpret_cast(reg_12_1[i]) = NEG_INFINITY; + reinterpret_cast(reg_12_3[i]) = 0.0f; + } + + //global_store_op 13 decls + int oob_M_13 = desc_max.dims[2]; + int row_13 = q_row_coord + local_wid * WARP_TILE_M_1 + (tiw / 4); + char *ptr_13 = reinterpret_cast(d_max) + + batch_coord_1 * desc_max.strides[0] * BYTES_PER_ELEMENT_13 + + head_coord_1 * desc_max.strides[1] * BYTES_PER_ELEMENT_13 + + row_13 * desc_max.strides[2] * BYTES_PER_ELEMENT_13; + + //global_store_op 14 decls + int oob_M_14 = desc_sum_exp.dims[2]; + int row_14 = q_row_coord + local_wid * WARP_TILE_M_1 + (tiw / 4); + char *ptr_14 = reinterpret_cast(d_sum_exp) + + batch_coord_1 * desc_sum_exp.strides[0] * BYTES_PER_ELEMENT_14 +)KERNEL" + R"KERNEL( + + head_coord_1 * desc_sum_exp.strides[1] * BYTES_PER_ELEMENT_14 + + row_14 * desc_sum_exp.strides[2] * BYTES_PER_ELEMENT_14; + + // shared_load_op 15 decls + + //global_load_shared_store_op 16 decls + + //mma_op 17 decls + + r32 reg_17_0[REGS_M_17][REGS_N_17][WARP_REGS_17]; + + //global_store_op 20 decls + static constexpr int cols_per_step_20 = (TILE_K_1 <= 128) ? TILE_K_1 : 64; + uint32_t smem_20 = smem_O + (local_gid * 4 + local_wid) * 16 * cols_per_step_20 * BYTES_PER_ELEMENT_1; + + const int swizzled_row_20 = (tiw % 16); + const int swizzled_col_20 = (tiw / 16); + constexpr int stsm_per_tile_20 = 128 / (BYTES_PER_BANK_1 * 2); + uint32_t stsm_base_20[stsm_per_tile_20]; + +#pragma unroll + for (int n = 0; n < stsm_per_tile_20; ++n) { + stsm_base_20[n] = smem_20 + ((swizzled_row_20 % 8) ^ (n * 2 + swizzled_col_20)) * BYTES_PER_BANK_1 + swizzled_row_20 * 128; + } + + const int row_20 = q_row_coord + (local_wid % 4) * 16; + + uint32_t local_smem_bar_tma_q = smem_bar_q_tma + buffer_id_q * 8; + uint32_t local_smem_bar_mma_q = smem_bar_q_mma + buffer_id_q * 8; + + if (local_gid == 1) { + named_barrier_arrive(SOFTMAX_2_BARRIER_1, 256); + } + + if (kv_loop_left_bound < kv_loop_right_bound) { + uint32_t smem_bar_phase_q = cnt_q < BUFFERS_Q_1 ? 0 : 1; + + cnt_q = cnt_q < (2 * BUFFERS_Q_1 - 1) ? (cnt_q + 1) : 0; + buffer_id_q = buffer_id_q < ( BUFFERS_Q_1 - 1) ? (buffer_id_q + 1) : 0; + + if (!smem_bar_peek(local_smem_bar_tma_q, smem_bar_phase_q)) { + smem_bar_wait(local_smem_bar_tma_q, smem_bar_phase_q); + } + } + + r32 reg_1_0[REGS_M_1][REGS_O_1][WARP_REGS_1]; + memset(®_1_0[0][0][0], 0, sizeof(reg_1_0)); + + named_barrier_wait(MATH_WORKGROUP_1, 256); +#pragma unroll 1 + for (int kv_loop = kv_loop_left_bound; kv_loop < kv_loop_right_bound; kv_loop++) { + static constexpr int kMulPipeCount = REGS_M_1 * REGS_N_1 * WARP_REGS_1; + + static constexpr int kSubtractPipeCount = 6; + static constexpr int kConvertPipeCount = 4; + static constexpr int oBlockSize = 1; + + const int p_col_coord = kv_loop * TILE_N_1; + const int causal_mask_col = p_col_coord + (tiw % 4) * 2; + const int causal_mask_row = q_row_coord + local_wid * 16 + (tiw / 4); + + uint32_t local_smem_bar_tma_k = smem_bar_k_tma + buffer_id_kv * 8; + uint32_t local_smem_bar_mma_k = smem_bar_k_mma + buffer_id_kv * 8; + + uint32_t local_smem_bar_tma_v = smem_bar_v_tma + buffer_id_kv * 8; + uint32_t local_smem_bar_mma_v = smem_bar_v_mma + buffer_id_kv * 8; + + uint32_t smem_bar_phase_kv = cnt_kv < BUFFERS_K_1 ? 0 : 1; + + cnt_kv = cnt_kv < (2 * BUFFERS_K_1 - 1) ? (cnt_kv + 1) : 0; + buffer_id_kv = buffer_id_kv < ( BUFFERS_K_1 - 1) ? (buffer_id_kv + 1) : 0; + + memset(®_8_0[0][0][0], 0, sizeof(reg_8_0)); + + if (!smem_bar_peek(local_smem_bar_tma_k, smem_bar_phase_kv)) { + smem_bar_wait(local_smem_bar_tma_k, smem_bar_phase_kv); + } + + warpgroup_arrive(); +#pragma unroll + for (int k = 0; k < REGS_K_1; ++k) { + if (k == REGS_K_1 - 1) { + BMM_S_GMMA_ISB(gmma_desc_q.desc, gmma_desc_k.desc, reinterpret_cast(reg_8_0[0])); + } else { + BMM_S_GMMA(gmma_desc_q.desc, gmma_desc_k.desc, reinterpret_cast(reg_8_0[0])); + } + int2 &tmp_desc_q = reinterpret_cast(gmma_desc_q.desc); + tmp_desc_q.x += BYTES_PER_GMMA_K_NO_4LSB_1; + if ((k % 4) == 3) { + tmp_desc_q.x -= BYTES_PER_GMMA_K_NO_4LSB_1 * 4; + tmp_desc_q.x += ((128*TILE_M_1*2) >> 4); + } + int2 &tmp_desc_k = reinterpret_cast(gmma_desc_k.desc); + tmp_desc_k.x += BYTES_PER_GMMA_K_NO_4LSB_1; + if ((k % 4) == 3) { + tmp_desc_k.x -= BYTES_PER_GMMA_K_NO_4LSB_1 * 4; + tmp_desc_k.x += ((128*TILE_N_1) >> 4); + } + } + { + int2 &tmp_desc_q = reinterpret_cast(gmma_desc_q.desc); + tmp_desc_q.x -= ((128 * TILE_M_1 * 2) >> 4) * (REGS_K_1 / 4); + int2 &tmp_desc_k = reinterpret_cast(gmma_desc_k.desc); + tmp_desc_k.x -= ((128 * TILE_N_1) >> 4) * (REGS_K_1 / 4); + } + gmma_desc_k.increment_smem_buffer(); + warpgroup_wait<0>(); + + if (local_wid == 0 && elect_one) { + smem_bar_arrive(local_smem_bar_mma_k); + } + + uint32_t wait_v_done = smem_bar_peek(local_smem_bar_tma_v, smem_bar_phase_kv); + + //mha_mask_op 11 code + if (kv_loop >= kv_loop_right_bound - 1) + { +#pragma unroll + for (int m = 0; m < REGS_M_11; ++m) { +#pragma unroll + for (int n = 0; n < REGS_N_11; ++n) { +#pragma unroll + for (int i = 0; i < WARP_REGS_11; ++i) { + int row = causal_mask_row + m * WARPS_PER_GROUP_0 * WARP_TILE_M_11 + ((i / 2) % 2) * 8; + int col = causal_mask_col + n * WARP_TILE_N_11 + i % 2 + (i / 4) * 8; + bool mask = compute_diagonal_band_mask_11(row, col, actual_seqlen_kv_1, actual_seqlen_q_1, shift_right_bound, left_bound); + if (!mask) { + reinterpret_cast(reg_8_0[m][n][i]) = neg_infinity; + } + } + } + } + } + + //global_load_op 10 code + + //mha_mask_op 11 code + if (kv_loop >= kv_loop_right_bound - 1) + { +#pragma unroll + for (int m = 0; m < REGS_M_11; ++m) { +#pragma unroll + for (int n = 0; n < REGS_N_11; ++n) { +#pragma unroll + for (int i = 0; i < WARP_REGS_11; ++i) { + int row = causal_mask_row + m * WARPS_PER_GROUP_0 * WARP_TILE_M_11 + ((i / 2) % 2) * 8; + int col = causal_mask_col + n * WARP_TILE_N_11 + i % 2 + (i / 4) * 8; + bool mask = compute_diagonal_band_mask_11(row, col, actual_seqlen_kv_1, actual_seqlen_q_1, shift_right_bound, left_bound); + if (!mask) { + reinterpret_cast(reg_8_0[m][n][i]) = neg_infinity; + } + } + } + } + } + + //softmax_op 12 code + static constexpr float inv_ln2_12 = 1.4426950408889634074f; + float scaled_inv_ln2_12 = inv_ln2_12 * reinterpret_cast(reg_10_0[0]); + + r32 *flattened_s = reinterpret_cast(reg_8_0); + r32 *flattened_s_fp16 = reinterpret_cast(reg_12_0); + r32 *flattened_o = reinterpret_cast(reg_1_0); + + float tmp_max_level_0[REGS_N_12][4]; + float tmp_max_level_1[REGS_N_12][2]; + float tmp_max_level_2[REGS_N_12 / 2][2]; +#pragma unroll + for (int m = 0; m < REGS_M_12; ++m) { +#pragma unroll + for (int n = 0; n < REGS_N_12; ++n) { +#pragma unroll + for (int i = 0; i < WARP_REGS_12; i+=2) { + tmp_max_level_0[n][i/2] = fmaxf(reinterpret_cast(reg_8_0[m][n][i+0]), + reinterpret_cast(reg_8_0[m][n][i+1])); + } + } + } + // FMUL +#pragma unroll + for (int i = 0; i < kMulPipeCount; i++) { + reinterpret_cast(flattened_s[i]) = __fmul_rn(scaled_inv_ln2_12, reinterpret_cast(flattened_s[i])); + } + { +#pragma unroll + for (int n = 0; n < REGS_N_12; ++n) { +#pragma unroll + for (int i = 0; i < 2; i++) { + tmp_max_level_1[n][i] = fmaxf(tmp_max_level_0[n][i + 0], + tmp_max_level_0[n][i + 2]); + } + } + } + { +#pragma unroll + for (int n = 0; n < REGS_N_12; n+=2) { + tmp_max_level_2[n/2][0] = fmaxf(tmp_max_level_1[n][0], tmp_max_level_1[n+1][0]); + tmp_max_level_2[n/2][1] = fmaxf(tmp_max_level_1[n][1], tmp_max_level_1[n+1][1]); + } + } + if (TILE_N_1 == 256) { + float tmp_max_level_3[REGS_N_12 / 4][2]; + float tmp_max_level_4[FORT_MAX(1,REGS_N_12 / 8)][2]; +#pragma unroll + for (int n = 0; n < REGS_N_12; n+=4) { + tmp_max_level_3[n/4][0] = fmaxf(tmp_max_level_2[n/2][0], tmp_max_level_2[n/2+1][0]); + tmp_max_level_3[n/4][1] = fmaxf(tmp_max_level_2[n/2][1], tmp_max_level_2[n/2+1][1]); + } +#pragma unroll + for (int n = 0; n < REGS_N_12; n+=8) { + tmp_max_level_4[n/8][0] = fmaxf(tmp_max_level_3[n/4][0], tmp_max_level_3[n/4+1][0]); + tmp_max_level_4[n/8][1] = fmaxf(tmp_max_level_3[n/4][1], tmp_max_level_3[n/4+1][1]); + } + reinterpret_cast(reg_12_2[0]) = fmaxf(tmp_max_level_4[0][0], tmp_max_level_4[1][0]); + reinterpret_cast(reg_12_2[1]) = fmaxf(tmp_max_level_4[0][1], tmp_max_level_4[1][1]); + } else if (TILE_N_1 == 128) { + float tmp_max_level_3[REGS_N_12 / 4][2]; +#pragma unroll + for (int n = 0; n < REGS_N_12; n+=4) { + tmp_max_level_3[n/4][0] = fmaxf(tmp_max_level_2[n/2][0], tmp_max_level_2[n/2+1][0]); + tmp_max_level_3[n/4][1] = fmaxf(tmp_max_level_2[n/2][1], tmp_max_level_2[n/2+1][1]); + } + reinterpret_cast(reg_12_2[0]) = fmaxf(tmp_max_level_3[0][0], tmp_max_level_3[1][0]); + reinterpret_cast(reg_12_2[1]) = fmaxf(tmp_max_level_3[0][1], tmp_max_level_3[1][1]); + } else { + reinterpret_cast(reg_12_2[0]) = fmaxf(tmp_max_level_2[0][0], tmp_max_level_2[1][0]); + reinterpret_cast(reg_12_2[1]) = fmaxf(tmp_max_level_2[0][1], tmp_max_level_2[1][1]); + } + + // Apply the functor for each row inside each group of 4 threads. +#pragma unroll + for (int m = 0; m < ROWS_PER_THREAD_12; ++m) { + r32 tmp_0 = __shfl_xor_sync(uint32_t(-1), reg_12_2[m], 1); + + reinterpret_cast(reg_12_2[m]) = fmaxf(reinterpret_cast(reg_12_2[m]), + reinterpret_cast(tmp_0)); + + tmp_0 = __shfl_xor_sync(uint32_t(-1), reg_12_2[m], 2); + + reinterpret_cast(reg_12_2[m]) = fmaxf(reinterpret_cast(reg_12_2[m]), + reinterpret_cast(tmp_0)) * scaled_inv_ln2_12; + } + + // Update acc_max scale of flash attention +#pragma unroll + for (int m = 0; m < ROWS_PER_THREAD_12; ++m) { + r32 curr_max = reg_12_1[m]; + reinterpret_cast(reg_12_1[m]) = fmaxf(reinterpret_cast(reg_12_2[m]), + reinterpret_cast(curr_max)); + reg_12_2[m] = curr_max; + } + + float alpha_12[ROWS_PER_THREAD_12]; +#pragma unroll + for (int m = 0; m < ROWS_PER_THREAD_12; ++m) { + float p_max = reinterpret_cast(reg_12_2[m]); + float acc_max = reinterpret_cast(reg_12_1[m]); + + // disable FMA for scale * P - max + float tmp = (acc_max == NEG_INFINITY) ? p_max : __fsub_rn(p_max, acc_max); + reinterpret_cast(alpha_12[m]) = fp32_exp2(reinterpret_cast(tmp)); + } + +#pragma unroll + for (int i = 0; i < ROWS_PER_THREAD_12; ++i) { + reinterpret_cast(reg_12_4[i]) = 0.0f; + } + + // Broadcast scaled_max and alpha + float flattened_scaled_max[REGS_M_12 * REGS_N_12 * WARP_REGS_12]; + float flattened_alpha[REGS_M_12 * REGS_O_1 * WARP_REGS_12]; +#pragma unroll + for (int m = 0; m < REGS_M_12; ++m) { +#pragma unroll + for (int n = 0; n < REGS_N_12; ++n) { +#pragma unroll + for (int i = 0; i < WARP_REGS_12; ++i) { + int row_indx = m * 2 + (i / 2) % 2; + float max = reinterpret_cast(reg_12_1[row_indx]); + flattened_scaled_max[m * REGS_N_12 * WARP_REGS_12 + n * WARP_REGS_12 + i] = (max == NEG_INFINITY ? 0.0f : max); + } + } + } + +#pragma unroll + for (int m = 0; m < REGS_M_12; ++m) { +#pragma unroll + for (int n = 0; n < REGS_O_1; ++n) { +#pragma unroll + for (int i = 0; i < WARP_REGS_12; ++i) { + int row_indx = m * 2 + (i / 2) % 2; + flattened_alpha[m * REGS_O_1 * WARP_REGS_12 + n * WARP_REGS_12 + i] = alpha_12[row_indx]; + } + } + } + + cfence(); + if (local_gid == 1) { + named_barrier_wait(SOFTMAX_1_BARRIER_1, 256); + } else { + named_barrier_wait(SOFTMAX_2_BARRIER_1, 256); + } + cfence(); + + // FSUB +#pragma unroll + for (int i = 0; i < kSubtractPipeCount; i++) { +)KERNEL" + R"KERNEL( + reinterpret_cast(flattened_s[i]) = __fsub_rn(reinterpret_cast(flattened_s[i]), flattened_scaled_max[i]); + } +#pragma unroll + + for (int i = 0; i < REGS_M_12 * REGS_N_12 * WARP_REGS_12; i+=2) { + int m_0 = (i-kConvertPipeCount+0) / (REGS_N_12 * WARP_REGS_12); + int ii_0 = (i-kConvertPipeCount+0) % WARP_REGS_12; + int m_1 = (i-kConvertPipeCount+1) / (REGS_N_12 * WARP_REGS_12); + int ii_1 = (i-kConvertPipeCount+1) % WARP_REGS_12; + int row_indx_0 = m_0 * 2 + (ii_0 / 2) % 2; + int row_indx_1 = m_1 * 2 + (ii_1 / 2) % 2; + // cast and FADD + if (i >= kConvertPipeCount) { + fp32x2_to_bf16x2(&flattened_s_fp16[(i - kConvertPipeCount)/2], &flattened_s[i - kConvertPipeCount]); + reinterpret_cast(reg_12_4[row_indx_0]) += reinterpret_cast(flattened_s[i-kConvertPipeCount+0]); + reinterpret_cast(reg_12_4[row_indx_1]) += reinterpret_cast(flattened_s[i-kConvertPipeCount+1]); + } + cfence(); + + // exp + flattened_s[i+0] = fp32_exp2(flattened_s[i+0]); + + // O update +#pragma unroll + for (int j = 0; j < oBlockSize; j++) { + if (oBlockSize*i+j < REGS_O_1 * REGS_M_1 * WARP_REGS_1) { + reinterpret_cast(flattened_o[oBlockSize*i+j]) *= flattened_alpha[oBlockSize*i+j]; + } + } + + // fmul + if (i + kMulPipeCount < REGS_M_12 * REGS_N_12 * WARP_REGS_12) { + reinterpret_cast(flattened_s[i+kMulPipeCount+0]) = __fmul_rn(scaled_inv_ln2_12, reinterpret_cast(flattened_s[i+kMulPipeCount+0])); + reinterpret_cast(flattened_s[i+kMulPipeCount+1]) = __fmul_rn(scaled_inv_ln2_12, reinterpret_cast(flattened_s[i+kMulPipeCount+1])); + } + // fsub + + if (i + kSubtractPipeCount < REGS_M_12 * REGS_N_12 * WARP_REGS_12) { + reinterpret_cast(flattened_s[i+kSubtractPipeCount+0]) = __fsub_rn(reinterpret_cast(flattened_s[i+kSubtractPipeCount+0]), flattened_scaled_max[i+kSubtractPipeCount+0]); + reinterpret_cast(flattened_s[i+kSubtractPipeCount+1]) = __fsub_rn(reinterpret_cast(flattened_s[i+kSubtractPipeCount+1]), flattened_scaled_max[i+kSubtractPipeCount+1]); + } + + // exp + flattened_s[i+1] = fp32_exp2(flattened_s[i+1]); + + // O update +#pragma unroll + for (int j = 0; j < oBlockSize; j++) { + if (oBlockSize+oBlockSize*i+j < REGS_O_1 * REGS_M_1 * WARP_REGS_1) { + reinterpret_cast(flattened_o[oBlockSize+oBlockSize*i+j]) *= flattened_alpha[oBlockSize+oBlockSize*i+j]; + } + } + } + + if (local_gid == 0) { + named_barrier_arrive(SOFTMAX_1_BARRIER_1, 256); + } else { + named_barrier_arrive(SOFTMAX_2_BARRIER_1, 256); + } + cfence(); + +#pragma unroll + for (int i = REGS_M_12 * REGS_N_12 * WARP_REGS_12 - kConvertPipeCount; i < REGS_M_12 * REGS_N_12 * WARP_REGS_12; i+=2) { + int m_0 = (i + 0) / (REGS_N_12 * WARP_REGS_12); + int ii_0 = (i + 0) % WARP_REGS_12; + int m_1 = (i + 1) / (REGS_N_12 * WARP_REGS_12); + int ii_1 = (i + 1) % WARP_REGS_12; + int row_indx_0 = m_0 * 2 + (ii_0 / 2) % 2; + int row_indx_1 = m_1 * 2 + (ii_1 / 2) % 2; + fp32x2_to_bf16x2(&flattened_s_fp16[i/2], &flattened_s[i]); + reinterpret_cast(reg_12_4[row_indx_0]) += reinterpret_cast(flattened_s[i+0]); + reinterpret_cast(reg_12_4[row_indx_1]) += reinterpret_cast(flattened_s[i+1]); + } + + if (!wait_v_done) { + smem_bar_wait(local_smem_bar_tma_v, smem_bar_phase_kv); + } + + warpgroup_arrive(); +#pragma unroll + for (int k = 0; k < TILE_N_1 / GMMA_TILE_K_1; ++k) { + if (k == (TILE_N_1 / GMMA_TILE_K_1) - 1) { + BMM_O_GMMA_ISB(reg_12_0[0][k], gmma_desc_v.desc, reinterpret_cast(reg_1_0[0])); + } else { + BMM_O_GMMA(reg_12_0[0][k], gmma_desc_v.desc, reinterpret_cast(reg_1_0[0])); + } + int2 &tmp_desc_v = reinterpret_cast(gmma_desc_v.desc); + tmp_desc_v.x += BYTES_PER_GMMA_K_NO_4LSB_TRANS_1; + } + { + int2 &tmp_desc_v = reinterpret_cast(gmma_desc_v.desc); + tmp_desc_v.x -= BYTES_PER_GMMA_K_NO_4LSB_TRANS_1 * (TILE_N_1 / GMMA_TILE_K_1); + } + gmma_desc_v.increment_smem_buffer(); + + warpgroup_wait<0>(); + if (local_wid == 0 && elect_one) { + smem_bar_arrive(local_smem_bar_mma_v); + } + + // Update acc_sum of flash attention +#pragma unroll + for (int m = 0; m < ROWS_PER_THREAD_12; ++m) { + float p_sum = reinterpret_cast(reg_12_4[m]); + float acc_sum = reinterpret_cast(reg_12_3[m]); + + reinterpret_cast(reg_12_3[m]) = reinterpret_cast(alpha_12[m]) * acc_sum + p_sum; + } + } + if (kv_loop_left_bound < kv_loop_right_bound) { + gmma_desc_q.increment_smem_buffer(); + + if (local_wid == 0 && elect_one) { + smem_bar_arrive(local_smem_bar_mma_q); + } + } + if (local_gid == 0) { + named_barrier_arrive(SOFTMAX_2_BARRIER_1, 256); + } +#pragma unroll + for (int m = 0; m < ROWS_PER_THREAD_12; ++m) { + r32 tmp_0 = __shfl_xor_sync(uint32_t(-1), reg_12_3[m], 1); + + reinterpret_cast(reg_12_3[m]) = reinterpret_cast(reg_12_3[m]) + + reinterpret_cast(tmp_0); + + tmp_0 = __shfl_xor_sync(uint32_t(-1), reg_12_3[m], 2); + + reinterpret_cast(reg_12_3[m]) = reinterpret_cast(reg_12_3[m]) + + reinterpret_cast(tmp_0); + + float sum = reinterpret_cast(reg_12_3[m]); + reinterpret_cast(reg_12_1[m]) *= ln2; + + reinterpret_cast(reg_12_5[m]) = (sum == 0.f) ? NEG_INFINITY : reinterpret_cast(reg_12_1[m]) + __logf(sum); + + beta_12[m] = (sum == 0.f) ? 0.f : 1.f / sum; + } + +#pragma unroll + for (int m = 0; m < REGS_M_1; ++m) { +#pragma unroll + for (int n = 0; n < REGS_O_1; ++n) { +#pragma unroll + for (int i = 0; i < WARP_REGS_1; ++i) { + int row_indx = m * 2 + (i / 2) % 2; + + reinterpret_cast(reg_1_0[m][n][i]) *= beta_12[row_indx]; + } + } + } + + //global_store_op 13 code +#pragma unroll + for (int m = 0; m < ROWS_PER_THREAD_13; ++m) { + const int row_offset = (m % 2) * THREADS_PER_WARP_TILE_M_13 + (m / 2) * WARPS_M_13 * WARP_TILE_M_13; + + if (row_13 + row_offset < oob_M_13 && (tiw % 4) == 0) { + stg_32(ptr_13 + row_offset * desc_max.strides[2] * BYTES_PER_ELEMENT_13, ®_12_1[m]); + } + } + + //global_store_op 14 code +#pragma unroll + for (int m = 0; m < ROWS_PER_THREAD_14; ++m) { + const int row_offset = (m % 2) * THREADS_PER_WARP_TILE_M_14 + (m / 2) * WARPS_M_14 * WARP_TILE_M_14; + + if (row_14 + row_offset < oob_M_14 && (tiw % 4) == 0) { + stg_32(ptr_14 + row_offset * desc_sum_exp.strides[2] * BYTES_PER_ELEMENT_14, ®_12_3[m]); + } + } + + tmastg_wait(); + + //global_store_op 20 code +#pragma unroll + for (int s = 0; s < TILE_O_1 / 64; ++s) { +#pragma unroll + for (int n = 0; n < stsm_per_tile_20; ++n) { + uint32_t tmp[4]; + fp32x2_to_bf16x2(&tmp[0], ®_1_0[0][s * stsm_per_tile_20 + n][0]); + fp32x2_to_bf16x2(&tmp[1], ®_1_0[0][s * stsm_per_tile_20 + n][2]); + fp32x2_to_bf16x2(&tmp[2], ®_1_0[0][s * stsm_per_tile_20 + n][4]); + fp32x2_to_bf16x2(&tmp[3], ®_1_0[0][s * stsm_per_tile_20 + n][6]); + stsm_x4(stsm_base_20[n] + s * 64 * 16 * BYTES_PER_ELEMENT_1, tmp); + } + } + fence_view_async_shared(); + + int head_coord_loc = 0; + int row = row_20; + int head = head_coord_1; +#pragma unroll + for (int s = 0; s < TILE_K_1 / 64; ++s) { + utmastg_4d_tiled(&tma_O, + smem_20 + s * 64 * 16 * BYTES_PER_ELEMENT_1, + s * 64, + row, + head + head_coord_loc, + batch_coord_1); + } + tmastg_arrive(); + + named_barrier_wait(TILE_ID_SYNC_3_BARRIER_1, THREADS_ON_TILE_BARRIER_1); + if (tiw == 0) { + lds_32(&tile_id, smem_tile_id_1); + } + named_barrier_arrive(TILE_ID_SYNC_2_BARRIER_1, THREADS_ON_TILE_BARRIER_1); + tile_id = __shfl_sync(0xffffffff, tile_id, 0); + } + tmastg_wait(); + reg_alloc<232>(); + } +} +)KERNEL"; +inline constexpr size_t d128_fprop_source_len = sizeof(d128_fprop_source) - 1; + +inline constexpr const char d128_fprop_flags[] = R"FLAGS(--gpu-architecture=sm_90a +--std=c++17 +--define-macro=__x86_64__ +-w +--define-macro=__CUDACC_RTC__ +-default-device +--use_fast_math +)FLAGS"; +inline constexpr size_t d128_fprop_flags_len = sizeof(d128_fprop_flags) - 1; + +} // namespace generated +} // namespace experimental +} // namespace cudnn_frontend + +#if defined(_MSC_VER) +#pragma warning(pop) +#endif + +#if defined(__clang__) +#pragma clang diagnostic pop +#endif diff --git a/deps/cudnn-frontend/include/cudnn_frontend/generated/sdpa/sm90/prefill/full_seqlens/d64_fprop_kernel.h b/deps/cudnn-frontend/include/cudnn_frontend/generated/sdpa/sm90/prefill/full_seqlens/d64_fprop_kernel.h new file mode 100644 index 0000000..92d8d1b --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/generated/sdpa/sm90/prefill/full_seqlens/d64_fprop_kernel.h @@ -0,0 +1,1763 @@ +// Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: MIT +#pragma once + +// Suppress clang warning for overlength strings (concatenated result may exceed 65536) +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Woverlength-strings" +#endif + +// Suppress MSVC warning C4068 (unknown pragma) if clang pragmas are seen +#if defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable : 4068) +#endif + +namespace cudnn_frontend { +namespace experimental { +namespace generated { + +inline constexpr const char d64_fprop_source[] = + R"KERNEL( +//receive_op 0 includes +typedef unsigned long long uint64_t; +typedef unsigned int uint32_t; +typedef unsigned short uint16_t; +typedef unsigned char uint8_t; +typedef long long int64_t; +typedef int int32_t; +typedef short int16_t; +typedef signed char int8_t; +typedef unsigned int r32; + +struct CUtensorMap { + alignas(64) uint64_t opaque[16]; +}; + +#define CUDACC_VERSION __CUDACC_VER_MAJOR__ * 10 + __CUDACC_VER_MINOR__ + +namespace fort { + static const uint64_t AMPERE_MEM_DESC_DEFAULT = uint64_t(0x1000000000000000ul); + static const uint64_t MEM_DESC_DEFAULT = AMPERE_MEM_DESC_DEFAULT; + static const uint32_t MMA_PEER_BIT_MASK = 0xFEFFFFFF; + +#define NEG_INFINITY __int_as_float(0xff800000) + + typedef uint16_t half_t; + typedef uint16_t bfloat16_t; + +#define FORT_MIN(a,b) ((a) < (b) ? (a) : (b)) +#define FORT_MAX(a,b) ((a) > (b) ? (a) : (b)) +#define FORT_DIV_UP(a,b) (((a) + (b) - 1) / (b)) +#define FORT_ROUND_UP(a,b) ((((a) + (b) - 1) / (b)) * (b)) + + typedef struct tensor_descriptor { + static const int MAX_DIMS = 12; + + int64_t num_dims; + int64_t dims[MAX_DIMS]; + int64_t strides[MAX_DIMS]; + } tensor_descriptor; + + typedef struct FastDivisor { + uint32_t val, shr, mul; + } FastDivisor_t; + + inline __device__ void + fastDivMod(const FastDivisor_t &d, uint32_t val, uint32_t &div, uint32_t &mod) { + div = __umulhi((uint32_t)2 * val, d.mul) >> d.shr; + mod = val - div * d.val; + } + __device__ __inline__ void cfence() {} + +#if __CUDA_ARCH__ >= 900 + + inline __device__ uint32_t elect_one_sync() { + uint32_t pred = 0; + uint32_t laneid = 0; + asm volatile( + "{\n" + ".reg .b32 %%rx;\n" + ".reg .pred %%px;\n" + " elect.sync %%rx|%%px, %2;\n" + "@%%px mov.s32 %1, 1;\n" + " mov.s32 %0, %%rx;\n" + "}\n" : "+r"(laneid), "+r"(pred) : "r"(0xFFFFFFFF)); + return pred; + } + + template + inline __device__ void reg_alloc() { + // const int TARGET_REG_COUNT = 232; // Example values (use with reg_delloc's VALUE together): 224, 232, 208, 216 + asm volatile("setmaxnreg.inc.sync.aligned.u32 %0;\n" : : "n"(TARGET_REG_COUNT)); + } + + template + inline __device__ void reg_dealloc() { + // const int TARGET_REG_COUNT = 40; // Exmaple values: 56, 40, 88, 72 + asm volatile("setmaxnreg.dec.sync.aligned.u32 %0;\n" : : "n"(TARGET_REG_COUNT)); + } + + inline __device__ void named_barrier_arrive(uint32_t BARRIER_ID, uint32_t NUM_THREADS) { + asm volatile("bar.arrive %0, %1;" : : "r"(BARRIER_ID), "r"(NUM_THREADS)); + } + + inline __device__ void named_barrier_wait(uint32_t BARRIER_ID, uint32_t NUM_THREADS) { + asm volatile("bar.sync %0, %1;" :: "r"(BARRIER_ID), "r"(NUM_THREADS)); + } + + inline __device__ void tmastg_arrive() { + asm volatile("cp.async.bulk.commit_group;"); + } + + inline __device__ void tmastg_wait() { + asm volatile("cp.async.bulk.wait_group.read %0;" : :"n"(0):"memory"); + } + +#endif + +#if defined( __CUDA_ARCH__ ) && __CUDA_ARCH__ >= 900 + inline __device__ void warpgroup_arrive() { + asm volatile("wgmma.fence.sync.aligned;\n" ::: "memory"); + } + template inline __device__ void warpgroup_wait() { + asm volatile("wgmma.wait_group.sync.aligned %0;\n" ::"n"(N) : "memory"); + } + inline __device__ void warpgroup_commit() { + asm volatile("wgmma.commit_group.sync.aligned;\n" ::: "memory"); + } + template + inline __device__ T &__wgmma_fence_operand(T &®) { + asm volatile("" : "+r"(reg)::"memory"); + return reg; + } +#endif + +#if defined( __CUDA_ARCH__ ) && __CUDA_ARCH__ >= 900 + inline __device__ void smem_bar_init(uint32_t smem_ptr, uint32_t thread_count) { + asm volatile ("mbarrier.init.shared::cta.b64 [%0], %1;\n" :: "r"(smem_ptr), "r"(thread_count)); + } + + inline __device__ void smem_bar_set_transaction_count(uint32_t smem_ptr, uint32_t expected_copy_bytes, uint32_t pred = 0) { + asm volatile("{\n\t.reg .pred p;" + " \n\tsetp.eq.u32 p, %2, 1;" + " \n\t@p mbarrier.arrive.expect_tx.shared::cta.b64 _, [%0], %1;" + " \n\t}" :: "r"(smem_ptr), "r"(expected_copy_bytes), "r"(pred)); + } + + inline __device__ uint32_t smem_bar_peek(uint32_t smem_ptr, uint32_t bar_phase) { + uint32_t bar_phase_out; + asm volatile("{\n\t.reg .pred P1;" + " \n\tmbarrier.try_wait.parity.shared::cta.b64 P1, [%1], %2;" + " \n\tselp.b32 %0, 1, 0, P1; \n\t}" :"=r" (bar_phase_out):"r"(smem_ptr),"r"(bar_phase)); + return bar_phase_out; + } + + inline __device__ void smem_bar_wait(uint32_t smem_ptr, uint32_t bar_phase) { + uint32_t large_val = 0x989680; + asm volatile("{\n\t.reg .pred P1;" + " \n\tLAB_WAIT:" + " \n\tmbarrier.try_wait.parity.shared::cta.b64 P1, [%0], %1, %2;" + " \n\t@P1 bra.uni DONE;" + " \n\tbra.uni LAB_WAIT;" + " \n\tDONE:" + " \n\t}" :: "r"(smem_ptr),"r"(bar_phase), "r"(large_val)); + } + + inline __device__ + void + wait_barrier(uint32_t smem_int_ptr, + int phase_bit) // Current phase bit the barrier waiting to flip + { + asm volatile( + "{\n" + ".reg .pred P1;\n" + "LAB_WAIT:\n" + "mbarrier.try_wait.parity.shared::cta.b64 P1, [%0], %1;\n" + "@P1 bra.uni DONE;\n" + "bra.uni LAB_WAIT;\n" + "DONE:\n" + "}\n" + :: "r"(smem_int_ptr), + "r"(phase_bit)); + } + + inline __device__ void smem_bar_arrive(uint32_t smem_ptr) { + asm volatile("{\n\t.reg .b64 state;" + " \n\tmbarrier.arrive.shared::cta.b64 state, [%0];" + " \n\t}" :: "r"(smem_ptr)); + } + + // Barrier arrive on local smem + inline __device__ + void arrive_barrier(uint32_t smem_addr) { + asm volatile( + "{\n\t" + "mbarrier.arrive.shared::cta.b64 _, [%0];\n\t" + "}" + : + : "r"(smem_addr)); + } + +#endif + + extern "C" __device__ uint32_t __nvvm_get_smem_pointer(void *ptr); + + inline __device__ uint32_t get_smem_pointer(const void *ptr) { + return __nvvm_get_smem_pointer(const_cast(ptr)); + } + + extern "C" __device__ inline void *__nv_cvta_shared_to_generic_impl(size_t __ptr) { + return (void *)(void __attribute__((address_space(3))) *)__ptr; + } + inline __device__ void *set_smem_pointer(uint32_t ptr) { + return __nv_cvta_shared_to_generic_impl(ptr); + } + + inline __device__ uint32_t + cast_smem_ptr_to_uint(void const* const ptr) { + return static_cast(__cvta_generic_to_shared(ptr)); + } + + inline __device__ void stsm_x4(r32 dst, r32 src[4]) { + asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" ::"r"(dst), "r"(src[0]), "r"(src[1]), "r"(src[2]), "r"(src[3])); + } + + inline __device__ void fence_view_async_shared(void){ + asm volatile ("fence.proxy.async.shared::cta;\n"); + } + + inline __device__ void sts_32(r32 dst, r32 src[1]) { + asm volatile("st.shared.b32 [%0], %1;\n" :: "r"(dst), "r"(src[0])); + } + + inline __device__ void lds_32(r32 dst[1], r32 src) { + asm volatile("ld.shared.b32 %0, [%1];\n" : "=r"(dst[0]) : "r"(src)); + } + + inline __device__ void stg_32(void *ptr, r32 val[1]) { + uint32_t *p = reinterpret_cast(ptr); + p[0] = val[0]; + } + + // Forward ReLU activations + + inline __device__ uint32_t fp32_exp2(uint32_t in) { + uint32_t out; + asm volatile("ex2.approx.ftz.f32 %0, %1;\n" : "=r"(out) : "r"(in)); + return out; + } + + inline __device__ void fp32x2_to_bf16x2(r32 dst[1], const r32 src[2]) { + asm volatile("cvt.rn.bf16x2.f32 %0, %1, %2;\n" : "=r"(dst[0]) : "r"(src[1]), "r"(src[0])); + } + + inline __device__ void bf16x2_to_fp32x2(r32 dst[2], const r32 src[1]) { + asm volatile( \ + "{\n" \ + " .reg .b16 lo, hi;\n" \ + " mov.b32 {lo, hi}, %2;\n" \ + " mov.b32 %0, {0, lo};\n" \ + " mov.b32 %1, {0, hi};\n" \ + "}\n" : "=r"(dst[0]), "=r"(dst[1]) : "r"(src[0])); + } + + //mma_pipeline_op 1 includes + typedef struct AttentionDescriptor { + // Input parameters + // b - batch + // q_h - num heads of q/dq/o/do + // k_h - num heads of k/dk + // v_h - num heads of v/dv + // s_q - max sequence length of q + // s_kv - max sequence length of kv + // d - hidden dim (head dim) + uint32_t b, q_h, k_h, v_h, s_q, s_kv, d_qk, d_v; + uint16_t q_heads_per_k, q_heads_per_v, min_q_heads_per_kv; + } AttentionDescriptor_t; + + // CUtensorMap, when issued using bulk copy async functions, are cached in constant cache. + // But the producer kernel may have directly written to global memory, without invalidating this constant cache. + // This acquire fence, invalidates the memory address in constant cache, using UTMACCTL.IV. + inline __device__ void tma_descriptor_fence_acquire( + CUtensorMap const* p_desc + ) { +#if (__CUDA_ARCH__ >= 900) && (CUDACC_VERSION >= 123) + uint64_t gmem_int_desc = reinterpret_cast(p_desc); + asm volatile ( + "fence.proxy.tensormap::generic.acquire.gpu [%0], 128;\n" + : + : "l"(gmem_int_desc) + : "memory"); + asm volatile ( + "cvta.global.u64 %0, %0;\n" + : + : "l"(gmem_int_desc), "l"(gmem_int_desc) + : "memory"); +#endif + } + + inline __device__ void utmaldg_4d_tiled ( + const void *p_desc, + uint32_t urb0, // smem offset + uint32_t urb1, // smem barrier offset + int32_t urb2, // m + int32_t urb3, // n + int32_t urb4, // b + int32_t urb5, // extra coord + const uint32_t elect_one, + const uint64_t mem_desc = MEM_DESC_DEFAULT + ) { + if (elect_one) { +#if __CUDA_ARCH__ >= 1200 && __CUDA_ARCH__ < 1300 + asm volatile( + "cp.async.bulk.tensor.4d.shared::cta.global.mbarrier::complete_tx::bytes.L2::cache_hint [%0], [%1, {%2, %3, %4, %5}], [%6], %7;\n" :: + "r"(urb0), + "l"(reinterpret_cast(p_desc)), + "r"(urb2), + "r"(urb3), + "r"(urb4), + "r"(urb5), + "r"(urb1), + "l"(mem_desc) : "memory" + ); +#else + asm volatile( + "cp.async.bulk.tensor.4d.shared::cluster.global.mbarrier::complete_tx::bytes.L2::cache_hint [%0], [%1, {%2, %3, %4, %5}], [%6], %7;\n" :: + "r"(urb0), + "l"(reinterpret_cast(p_desc)), + "r"(urb2), + "r"(urb3), + "r"(urb4), + "r"(urb5), + "r"(urb1), + "l"(mem_desc) : "memory" + ); +#endif + } + } + + inline __device__ void utmastg_4d_tiled( + const void *p_desc, + uint32_t smem_offset, + int32_t urb0, // m + int32_t urb1, // n + int32_t urb2, // h + int32_t urb3, // b + uint64_t mem_desc = MEM_DESC_DEFAULT + ) { + asm volatile( + "cp.async.bulk.tensor.4d.global.shared::cta.bulk_group [%0, {%1, %2, %3, %4}], [%5];\n" :: + "l"(reinterpret_cast(p_desc)), + "r"(urb0), + "r"(urb1), + "r"(urb2), + "r"(urb3), + "r"(smem_offset) : "memory" + ); + } + + inline __device__ void ldgdepbar() { + asm volatile("cp.async.commit_group;\n" ::); + } + + template + inline __device__ void depbar() { + asm volatile("cp.async.wait_group %0;\n" :: "n"(STAGES)); + } + + //mma_loop_op 2 includes + + //global_load_shared_store_op 4 includes + + // shared_load_op 5 includes + + // shared_load_op 6 includes + + //global_load_shared_store_op 7 includes + + //mma_op 8 includes +)KERNEL" + R"KERNEL( +// === Section: GMMA Operations === + inline __device__ void bf16gmma_fp32_64x128x16_TN_asmem_bsmem_isb(const uint64_t &desc_a, const uint64_t &desc_b, r32 acc[64]) { + __wgmma_mma_async_bf16(64, 128, 16, false, false, false, false, true, acc, desc_a, desc_b); + warpgroup_commit(); + } + + // === Section: GMMA Operations === + inline __device__ void bf16gmma_fp32_64x128x16_TN_asmem_bsmem(const uint64_t &desc_a, const uint64_t &desc_b, r32 acc[64]) { + __wgmma_mma_async_bf16(64, 128, 16, false, false, false, false, true, acc, desc_a, desc_b); + } + +#define BMM_S_GMMA bf16gmma_fp32_64x128x16_TN_asmem_bsmem +#define BMM_S_GMMA_ISB bf16gmma_fp32_64x128x16_TN_asmem_bsmem_isb +#define BMM_S_GMMA_C0 bf16gmma_fp32_64x128x16_TN_asmem_bsmem_ignoreC + + //pointwise_calc_op 9 includes + + //global_load_op 10 includes + + //softmax_op 12 includes + + // global_store_op 13 includes + + // global_store_op 14 includes + + // shared_load_op 15 includes + + //global_load_shared_store_op 16 includes + + //mma_op 17 includes + inline __device__ void bf16gmma_fp32_64x64x16_TT_arf_bsmem_isb(const r32 a[4], const uint64_t &desc_b, r32 acc[32]) { + __wgmma_mma_async_bf16(64, 64, 16, false, false, true, false, true, acc, a, desc_b); + warpgroup_commit(); + } + + inline __device__ void bf16gmma_fp32_64x64x16_TT_arf_bsmem(const r32 a[4], const uint64_t &desc_b, r32 acc[32]) { + __wgmma_mma_async_bf16(64, 64, 16, false, false, true, false, true, acc, a, desc_b); + } + +#define BMM_O_GMMA bf16gmma_fp32_64x64x16_TT_arf_bsmem +#define BMM_O_GMMA_ISB bf16gmma_fp32_64x64x16_TT_arf_bsmem_isb +#define BMM_O_GMMA_C0 bf16gmma_fp32_64x64x16_TT_arf_bsmem_ignoreC + + // global_store_op 20 includes + + //output_loop_op 3 includes + + // shared_store_op 18 includes + + // shared_load_op 19 includes +} // namespace fort + +//receive_op 0 types +using namespace fort; + +static constexpr int THREADS_PER_WARP_0 = 32; +static constexpr int WARPS_PER_GROUP_0 = FORT_MIN(4, 384 / THREADS_PER_WARP_0); +static constexpr int THREADS_PER_WARP_GROUP_0 = THREADS_PER_WARP_0 * WARPS_PER_GROUP_0; +static constexpr int BITS_PER_REGISTER_0 = 32; // NOTE: sizeof(uint) +static constexpr int BYTES_PER_REGISTER_0 = BITS_PER_REGISTER_0 / 8; +static constexpr int THREADS_PER_GROUP_0 = WARPS_PER_GROUP_0 * THREADS_PER_WARP_0; + +// mma_pipeline_op 1 types +static constexpr int BYTES_PER_BANK_1 = 16; +static constexpr int BITS_PER_ELEMENT_1 = 16; +static constexpr int ELEMS_PER_BANK_1 = BYTES_PER_BANK_1 * 8 / BITS_PER_ELEMENT_1; +static constexpr int BYTES_PER_ELEMENT_1 = BITS_PER_ELEMENT_1 / 8; +static constexpr int BYTES_PER_FP16_1 = 2; +static constexpr int ELEMS_PER_VECTOR_1 = BYTES_PER_REGISTER_0 / BYTES_PER_ELEMENT_1; +static constexpr int BITS_PER_ACC_1 = 32; +static constexpr int BYTES_PER_ACC_1 = BITS_PER_ACC_1 / 8; +static constexpr int TILE_M_1 = 64; +static constexpr int TILE_N_1 = 128; +static constexpr int TILE_K_1 = 64; +static constexpr int TILE_O_1 = 64; +static constexpr int TILE_V_1 = TILE_K_1; +static constexpr int TILE_DV_1 = 64; +static constexpr int WARP_TILE_M_1 = 16; +static constexpr int WARP_TILE_N_1 = 16; +static constexpr int WARP_TILE_K_1 = 32 / BYTES_PER_ELEMENT_1; +static constexpr int WARPS_M_1 = 64 / 16; +static constexpr int WARPS_N_1 = 16 / 16; +static constexpr int WARPS_K_1 = 16 / 16; +static constexpr int STAGES_1 = 1; +static constexpr int GROUPS_M_1 = 1; +static constexpr int GMMA_TILE_M_1 = WARPS_M_1 * WARP_TILE_M_1; +static constexpr int GMMA_TILE_K_1 = WARPS_K_1 * WARP_TILE_K_1; +static constexpr int WARP_TILES_M_1 = TILE_M_1 / WARP_TILE_M_1; +static constexpr int WARP_TILES_N_1 = TILE_N_1 / WARP_TILE_N_1; +static constexpr int WARP_TILES_K_1 = TILE_K_1 / WARP_TILE_K_1; +static constexpr int WARP_REGS_1 = 8; +static constexpr int REGS_M_1 = WARP_TILES_M_1 / WARPS_M_1; +static constexpr int REGS_N_1 = WARP_TILES_N_1 / WARPS_N_1; +static constexpr int REGS_K_1 = WARP_TILES_K_1 / WARPS_K_1; +static constexpr int REGS_K_FOR_DP_1 = TILE_V_1 / 16; +static constexpr int REGS_dQ_1 = TILE_O_1 / 16; + +static constexpr int REGS_O_1 = 4; + +static constexpr int CGA_M_1 = 1; +static constexpr int CGA_N_1 = 1; +static constexpr int CTA_MMA_1 = 1; +static constexpr int M_TILES_PER_OUTPUT_TILE_1 = 4; +static constexpr int N_TILES_PER_OUTPUT_TILE_1 = 1; + +static constexpr int CTA_TILE_M_1 = TILE_M_1 * GROUPS_M_1; +static constexpr int CTA_TILE_N_1 = 64; +static constexpr int CTA_TILE_K_1 = 128; +static constexpr int BUFFERS_Q_1 = 2; +static constexpr int BUFFERS_K_1 = 2; +static constexpr int BUFFERS_V_1 = 2; +static constexpr int BUFFERS_O_1 = 1; +static constexpr int BUFFERS_D_1 = 1; +static constexpr int SMEM_Q_1 = TILE_M_1 * TILE_K_1 * BYTES_PER_ELEMENT_1 * 2; +static constexpr int SMEM_K_1 = TILE_N_1 * TILE_K_1 * BYTES_PER_ELEMENT_1; +static constexpr int SMEM_V_1 = TILE_N_1 * TILE_K_1 * BYTES_PER_ELEMENT_1; +static constexpr int SMEM_PAGE_TABLE_1 = FORT_ROUND_UP(TILE_N_1 * 2 * 2 * sizeof(int), 1024); +static constexpr int SMEM_OFFSET_Q_1 = 1024 + (false ? 0 : SMEM_PAGE_TABLE_1); +static constexpr int SMEM_OFFSET_K_1 = SMEM_OFFSET_Q_1 + SMEM_Q_1 * BUFFERS_Q_1; +static constexpr int SMEM_OFFSET_V_1 = SMEM_OFFSET_K_1 + SMEM_K_1 * BUFFERS_K_1; +static constexpr int SMEM_OFFSET_O_1 = SMEM_OFFSET_V_1 + SMEM_V_1 * BUFFERS_V_1; +static constexpr int SMEM_OFFSET_D_1 = SMEM_OFFSET_O_1; +static constexpr int SMEM_OFFSET_BAND_BIAS_1 = SMEM_OFFSET_O_1; + +static constexpr int THREADS_ON_TILE_BARRIER_1 = 256+32+0; +static constexpr int TILE_ID_SYNC_2_BARRIER_1 = 2; +static constexpr int TILE_ID_SYNC_3_BARRIER_1 = 3; +static constexpr int SOFTMAX_1_BARRIER_1 = 4; +static constexpr int SOFTMAX_2_BARRIER_1 = 5; +static constexpr int MATH_WORKGROUP_1 = 6; +static constexpr int TMA_SYNC_BARRIER_1 = 7; + +static constexpr uint32_t BYTES_PER_GMMA_K_1 = GMMA_TILE_K_1 * BYTES_PER_ELEMENT_1; +static constexpr uint32_t BYTES_PER_GMMA_K_NO_4LSB_1 = BYTES_PER_GMMA_K_1 >> 4; +static constexpr uint32_t BYTES_PER_GMMA_K_TRANS_1 = GMMA_TILE_K_1 * 128; +static constexpr uint32_t BYTES_PER_GMMA_K_NO_4LSB_TRANS_1 = BYTES_PER_GMMA_K_TRANS_1 >> 4; + +static constexpr uint32_t GMMA_DESCRIPTOR_SWIZZLE_MODE_BITS_1 = 2; +static constexpr uint32_t GMMA_DESCRIPTOR_SWIZZLE_MODE_SHIFT_1 = 62; // bits 63-62 +static constexpr uint64_t GMMA_DESCRIPTOR_SWIZZLE_MODE_VALUE_Q_1 = 1; // SWIZZLE_NONE=0, SWIZZLE_128B=1, SWIZZLE_64B=2, SWIZZLE_32B=3 +static constexpr uint64_t GMMA_DESCRIPTOR_SWIZZLE_MODE_IN_BIT_LOCATION_Q_1 = (GMMA_DESCRIPTOR_SWIZZLE_MODE_VALUE_Q_1 & +((1u << GMMA_DESCRIPTOR_SWIZZLE_MODE_BITS_1) - 1)) +<< GMMA_DESCRIPTOR_SWIZZLE_MODE_SHIFT_1; +static constexpr uint64_t GMMA_DESCRIPTOR_SWIZZLE_MODE_VALUE_K_1 = 1; // SWIZZLE_NONE=0, SWIZZLE_128B=1, SWIZZLE_64B=2, SWIZZLE_32B=3 +static constexpr uint64_t GMMA_DESCRIPTOR_SWIZZLE_MODE_IN_BIT_LOCATION_K_1 = (GMMA_DESCRIPTOR_SWIZZLE_MODE_VALUE_K_1 & +((1u << GMMA_DESCRIPTOR_SWIZZLE_MODE_BITS_1) - 1)) +<< GMMA_DESCRIPTOR_SWIZZLE_MODE_SHIFT_1; +static constexpr uint64_t GMMA_DESCRIPTOR_SWIZZLE_MODE_VALUE_V_1 = 1; // SWIZZLE_NONE=0, SWIZZLE_128B=1, SWIZZLE_64B=2, SWIZZLE_32B=3 +static constexpr uint64_t GMMA_DESCRIPTOR_SWIZZLE_MODE_IN_BIT_LOCATION_V_1 = (GMMA_DESCRIPTOR_SWIZZLE_MODE_VALUE_V_1 & +((1u << GMMA_DESCRIPTOR_SWIZZLE_MODE_BITS_1) - 1)) +<< GMMA_DESCRIPTOR_SWIZZLE_MODE_SHIFT_1; + +static constexpr uint32_t GMMA_DESCRIPTOR_STRIDE_BYTE_OFFSET_SHIFT_1 = 32; // bits 45-32 +static constexpr uint64_t BYTES_PER_LEADING_DIM_1 = 128; +static constexpr uint64_t STRIDE_BYTE_OFFSET_Q_1 = (BYTES_PER_LEADING_DIM_1 * 8) / 16; // SWIZZLE_NONE=0, SWIZZLE_128B=8, SWIZZLE_64B=4, SWIZZLE_32B=2 +static constexpr uint64_t GMMA_DESCRIPTOR_STRIDE_BYTE_OFFSET_IN_BIT_LOCATION_Q_1 = STRIDE_BYTE_OFFSET_Q_1 << GMMA_DESCRIPTOR_STRIDE_BYTE_OFFSET_SHIFT_1; +static constexpr uint64_t STRIDE_BYTE_OFFSET_K_1 = (BYTES_PER_LEADING_DIM_1 * 8) / 16; // SWIZZLE_NONE=0, SWIZZLE_128B=8, SWIZZLE_64B=4, SWIZZLE_32B=2 +static constexpr uint64_t GMMA_DESCRIPTOR_STRIDE_BYTE_OFFSET_IN_BIT_LOCATION_K_1 = STRIDE_BYTE_OFFSET_K_1 << GMMA_DESCRIPTOR_STRIDE_BYTE_OFFSET_SHIFT_1; +static constexpr uint64_t STRIDE_BYTE_OFFSET_V_1 = (BYTES_PER_LEADING_DIM_1 * 8) / 16; // SWIZZLE_NONE=0, SWIZZLE_128B=8, SWIZZLE_64B=4, SWIZZLE_32B=2 +static constexpr uint64_t GMMA_DESCRIPTOR_STRIDE_BYTE_OFFSET_IN_BIT_LOCATION_V_1 = STRIDE_BYTE_OFFSET_V_1 << GMMA_DESCRIPTOR_STRIDE_BYTE_OFFSET_SHIFT_1; + +static constexpr uint32_t GMMA_DESCRIPTOR_LD_BYTE_OFFSET_SHIFT_1 = 16; // bits 29-16 +static constexpr uint32_t GMMA_DESCRIPTOR_LD_BYTE_OFFSET_VALUE_Q_1 = 128 * 64 / 16; // Not used??? +static constexpr uint64_t GMMA_DESCRIPTOR_LD_BYTE_OFFSET_IN_BIT_LOCATION_Q_1 = GMMA_DESCRIPTOR_LD_BYTE_OFFSET_VALUE_Q_1 +<< GMMA_DESCRIPTOR_LD_BYTE_OFFSET_SHIFT_1; +static constexpr uint32_t GMMA_DESCRIPTOR_LD_BYTE_OFFSET_VALUE_K_1 = 128 * 32 / 16; // Not used??? +static constexpr uint64_t GMMA_DESCRIPTOR_LD_BYTE_OFFSET_IN_BIT_LOCATION_K_1 = GMMA_DESCRIPTOR_LD_BYTE_OFFSET_VALUE_K_1 +<< GMMA_DESCRIPTOR_LD_BYTE_OFFSET_SHIFT_1; +static constexpr uint32_t GMMA_DESCRIPTOR_LD_BYTE_OFFSET_VALUE_V_1 = 128 * TILE_N_1 / 16; +static constexpr uint64_t GMMA_DESCRIPTOR_LD_BYTE_OFFSET_IN_BIT_LOCATION_V_1 = GMMA_DESCRIPTOR_LD_BYTE_OFFSET_VALUE_V_1 +<< GMMA_DESCRIPTOR_LD_BYTE_OFFSET_SHIFT_1; + +inline __device__ constexpr uint64_t create_gmma_desc_q_1() { + return ((uint64_t)0 | GMMA_DESCRIPTOR_SWIZZLE_MODE_IN_BIT_LOCATION_Q_1) + | GMMA_DESCRIPTOR_STRIDE_BYTE_OFFSET_IN_BIT_LOCATION_Q_1 + | GMMA_DESCRIPTOR_LD_BYTE_OFFSET_IN_BIT_LOCATION_Q_1; +} + +inline __device__ constexpr uint64_t create_gmma_desc_k_1() { + return ((uint64_t)0 | GMMA_DESCRIPTOR_SWIZZLE_MODE_IN_BIT_LOCATION_K_1) + | GMMA_DESCRIPTOR_STRIDE_BYTE_OFFSET_IN_BIT_LOCATION_K_1 + | GMMA_DESCRIPTOR_LD_BYTE_OFFSET_IN_BIT_LOCATION_K_1; +} + +// === Section: GMMA Descriptors & Mask Helpers === +)KERNEL" + R"KERNEL( +// === Section: GMMA Descriptors & Mask Helpers === +inline __device__ constexpr uint64_t create_gmma_desc_v_1() { + return ((uint64_t)0 | GMMA_DESCRIPTOR_SWIZZLE_MODE_IN_BIT_LOCATION_V_1) + | GMMA_DESCRIPTOR_STRIDE_BYTE_OFFSET_IN_BIT_LOCATION_V_1 + | GMMA_DESCRIPTOR_LD_BYTE_OFFSET_IN_BIT_LOCATION_V_1; +} + +class Gmma_descriptor { + public: + inline __device__ Gmma_descriptor(uint64_t desc_) { + desc = desc_; + } + template + inline __device__ void set_smem(uint32_t smem) { + desc |= (static_cast(smem & 0xFFFFFF) >> 4); + + max_desc = desc + (BYTES_PER_BUFFER >> 4) * (BUFFER_COUNT - 1); + } + template + inline __device__ void increment_smem_buffer() { + int2 &tmp = reinterpret_cast(desc); + tmp.x += (desc >= max_desc) ? -(BYTES_PER_BUFFER >> 4) * (BUFFER_COUNT - 1) + : (BYTES_PER_BUFFER >> 4); + } + + uint64_t desc; + uint64_t max_desc; +}; + +//global_load_shared_store_op 4 types +static constexpr int BITS_PER_ELEMENT_4 = 16; +static constexpr int BYTES_PER_ELEMENT_4 = BITS_PER_ELEMENT_4 / 8; +static constexpr int BYTES_PER_ACCESS_4 = 16; +static constexpr int TILE_M_4 = 64; +static constexpr int TILE_N_4 = 64; +static constexpr int BYTES_PER_SMEM_4 = TILE_M_4 * TILE_N_4 * BYTES_PER_ELEMENT_4; + +// shared_load_op 5 types +static constexpr int BITS_PER_ELEMENT_5 = 16; +static constexpr int BYTES_PER_ELEMENT_5 = 2; +static constexpr int BYTES_PER_QUAD_5 = 16; +static constexpr int WARP_TILE_M_5 = 16; +static constexpr int WARP_TILE_N_5 = 16; +static constexpr int TILE_M_5 = 64; +static constexpr int TILE_N_5 = 64; +static constexpr int BYTES_PER_SMEM_5 = TILE_M_5 * TILE_N_5 * BYTES_PER_ELEMENT_5; +static constexpr int WARPS_M_5 = 4; +static constexpr int WARPS_N_5 = 1; +static constexpr int WARP_TILES_M_5 = TILE_M_5 / WARP_TILE_M_5; +static constexpr int WARP_TILES_N_5 = TILE_N_5 / WARP_TILE_N_5; +static constexpr int THREADS_PER_WARP_TILE_M_5 = 8; +static constexpr int THREADS_PER_WARP_TILE_N_5 = 4; +static constexpr int REGS_M_5 = WARP_TILES_M_5 / WARPS_M_5; +static constexpr int REGS_N_5 = WARP_TILES_N_5 / WARPS_N_5; +static constexpr int BYTES_PER_LD_5 = 128; +static constexpr int SWIZZLE_SCALE_5 = FORT_MIN(BYTES_PER_LD_5 / 16, WARPS_PER_GROUP_0 * 4); + +// shared_load_op 6 types +static constexpr int BITS_PER_ELEMENT_6 = 16; +static constexpr int BYTES_PER_ELEMENT_6 = 2; +static constexpr int BYTES_PER_QUAD_6 = 16; +static constexpr int WARP_TILE_M_6 = 16; +static constexpr int WARP_TILE_N_6 = 16; +static constexpr int TILE_M_6 = 128; +static constexpr int TILE_N_6 = 64; + +static constexpr int BYTES_PER_SMEM_6 = TILE_M_6 * TILE_N_6 * BYTES_PER_ELEMENT_6; +static constexpr int WARPS_M_6 = 1; +static constexpr int WARPS_N_6 = 1; +static constexpr int WARP_TILES_M_6 = TILE_M_6 / WARP_TILE_M_6; +static constexpr int WARP_TILES_N_6 = TILE_N_6 / WARP_TILE_N_6; +static constexpr int THREADS_PER_WARP_TILE_M_6 = 8; +static constexpr int THREADS_PER_WARP_TILE_N_6 = 4; +static constexpr int REGS_M_6 = WARP_TILES_M_6 / WARPS_M_6; +static constexpr int REGS_N_6 = WARP_TILES_N_6 / WARPS_N_6; +static constexpr int BYTES_PER_LD_6 = 128; +static constexpr int SWIZZLE_SCALE_6 = FORT_MIN(BYTES_PER_LD_6 / 16, WARPS_PER_GROUP_0 * 4); + +//global_load_shared_store_op 7 types +static constexpr int BITS_PER_ELEMENT_7 = 16; +static constexpr int BYTES_PER_ELEMENT_7 = BITS_PER_ELEMENT_7 / 8; +static constexpr int BYTES_PER_ACCESS_7 = 16; +static constexpr int TILE_M_7 = 128; +static constexpr int TILE_N_7 = 64; +static constexpr int BYTES_PER_SMEM_7 = TILE_M_7 * TILE_N_7 * BYTES_PER_ELEMENT_7; + +//mma_op 8 types +static constexpr int BYTES_PER_ELEMENT_8 = 2; +static constexpr int BYTES_PER_ACC_8 = 4; +static constexpr int WARP_TILE_M_8 = 16; +static constexpr int WARP_TILE_N_8 = 16; +static constexpr int WARP_TILE_K_8 = 16; +static constexpr int TILE_M_8 = 64; +static constexpr int TILE_N_8 = 128; +static constexpr int TILE_K_8 = 64; +static constexpr int WARPS_M_8 = 4; +static constexpr int WARPS_N_8 = 1; +static constexpr int WARP_TILES_M_8 = TILE_M_8 / WARP_TILE_M_8; +static constexpr int WARP_TILES_N_8 = TILE_N_8 / WARP_TILE_N_8; +static constexpr int MMA_STEPS_K_8 = TILE_K_8 / WARP_TILE_K_8; +static constexpr int WARP_REGS_8 = 8; +static constexpr int REGS_M_8 = WARP_TILES_M_8 / WARPS_M_8; +static constexpr int REGS_N_8 = WARP_TILES_N_8 / WARPS_N_8; + +//pointwise_calc_op 9 types +static constexpr int WARP_TILE_M_9 = 16; +static constexpr int WARP_TILE_N_9 = 16; +static constexpr int TILE_M_9 = 64; +static constexpr int TILE_N_9 = 128; +static constexpr int WARPS_M_9 = 4; +static constexpr int WARPS_N_9 = 1; +static constexpr int WARP_TILES_M_9 = TILE_M_9 / WARP_TILE_M_9; +static constexpr int WARP_TILES_N_9 = TILE_N_9 / WARP_TILE_N_9; +static constexpr int WARP_REGS_9 = 8; +static constexpr int REGS_M_9 = WARP_TILES_M_9 / WARPS_M_9; +static constexpr int REGS_N_9 = WARP_TILES_N_9 / WARPS_N_9; + +//global_load_op 10 types +static constexpr int BITS_PER_ELEMENT_10 = 32; +static constexpr int BYTES_PER_ELEMENT_10 = 4; +static constexpr int REGISTERS_PER_VECTOR_10 = 1; +static constexpr int REGISTERS_PER_ACCESS_10 = 1; + +//mha_mask_op 11 types +static constexpr int WARP_TILE_M_11 = 16; +static constexpr int WARP_TILE_N_11 = 16; +static constexpr int TILE_M_11 = 64; +static constexpr int TILE_N_11 = 128; +static constexpr int WARPS_M_11 = 4; +static constexpr int WARPS_N_11 = 1; +static constexpr int WARP_TILES_M_11 = TILE_M_11 / WARP_TILE_M_11; +static constexpr int WARP_TILES_N_11 = TILE_N_11 / WARP_TILE_N_11; +static constexpr int WARP_REGS_11 = 8; +static constexpr int REGS_M_11 = WARP_TILES_M_11 / WARPS_M_11; +static constexpr int REGS_N_11 = WARP_TILES_N_11 / WARPS_N_11; +inline __device__ bool compute_diagonal_band_mask_11(const int row, const int col, const int actual_seqlen_kv, const int actual_seqlen_q, const int shift_right_bound, const int left_bound) { + constexpr bool is_bottom_right_alignment = false; + constexpr bool is_shift_right_bound = false; + constexpr bool is_right_bound = true; + constexpr bool is_left_bound = false; + constexpr bool need_oob_check = false; + + const int diag = is_bottom_right_alignment ? row + (actual_seqlen_kv - actual_seqlen_q) : row; + const int shifted_right_bound = is_shift_right_bound ? diag + shift_right_bound : diag; + const bool right_bound_mask = is_right_bound ? col <= shifted_right_bound : true; + const bool left_bound_mask = is_left_bound ? col + left_bound > diag : true; + const bool oob_check = need_oob_check ? (row < actual_seqlen_q && col < actual_seqlen_kv) : true; + return right_bound_mask && left_bound_mask && oob_check; +} + +//softmax_op 12 types +static constexpr int BYTES_PER_ELEMENT_12 = 2; +static constexpr int WARP_TILE_M_12 = 16; +static constexpr int WARP_TILE_N_12 = 16; +static constexpr int TILE_M_12 = 64; +static constexpr int TILE_N_12 = 128; +static constexpr int WARPS_M_12 = 4; +static constexpr int WARPS_N_12 = 1; +static constexpr int WARP_TILES_M_12 = TILE_M_12 / WARP_TILE_M_12; +static constexpr int WARP_TILES_N_12 = TILE_N_12 / WARP_TILE_N_12; +static constexpr int THREADS_PER_WARP_TILE_M_12 = 8; + +static constexpr int THREADS_PER_WARP_TILE_N_12 = 4; +static constexpr int WARP_REGS_12 = 8; +static constexpr int REGS_M_12 = WARP_TILES_M_12 / WARPS_M_12; +static constexpr int REGS_N_12 = WARP_TILES_N_12 / WARPS_N_12; +static constexpr int ROWS_PER_THREAD_12 = TILE_M_12 / WARPS_M_12 / THREADS_PER_WARP_TILE_M_12; + +// global_store_op 13 types +static constexpr int BYTES_PER_ELEMENT_13 = 4; +static constexpr int WARP_TILE_M_13 = 16; +static constexpr int WARP_TILE_N_13 = 16; +static constexpr int TILE_M_13 = 64; +static constexpr int TILE_N_13 = 128; +static constexpr int WARPS_M_13 = 4; +static constexpr int WARPS_N_13 = 1; +static constexpr int WARP_TILES_M_13 = TILE_M_13 / WARP_TILE_M_13; +static constexpr int WARP_TILES_N_13 = TILE_N_13 / WARP_TILE_N_13; +static constexpr int THREADS_PER_WARP_TILE_M_13 = 8; +static constexpr int THREADS_PER_WARP_TILE_N_13 = 4; +static constexpr int ROWS_PER_THREAD_13 = TILE_M_13 / WARPS_M_13 / THREADS_PER_WARP_TILE_M_13; + +// global_store_op 14 types +static constexpr int BYTES_PER_ELEMENT_14 = 4; +static constexpr int WARP_TILE_M_14 = 16; +static constexpr int WARP_TILE_N_14 = 16; +static constexpr int TILE_M_14 = 64; +static constexpr int TILE_N_14 = 128; +static constexpr int WARPS_M_14 = 4; +static constexpr int WARPS_N_14 = 1; +static constexpr int WARP_TILES_M_14 = TILE_M_14 / WARP_TILE_M_14; +static constexpr int WARP_TILES_N_14 = TILE_N_14 / WARP_TILE_N_14; +static constexpr int THREADS_PER_WARP_TILE_M_14 = 8; +static constexpr int THREADS_PER_WARP_TILE_N_14 = 4; +static constexpr int ROWS_PER_THREAD_14 = TILE_M_14 / WARPS_M_14 / THREADS_PER_WARP_TILE_M_14; + +// shared_load_op 15 types +static constexpr int BITS_PER_ELEMENT_15 = 16; +static constexpr int BYTES_PER_ELEMENT_15 = 2; +static constexpr int BYTES_PER_QUAD_15 = 16; +static constexpr int WARP_TILE_M_15 = 16; +static constexpr int WARP_TILE_N_15 = 16; +static constexpr int TILE_M_15 = 128; +static constexpr int TILE_N_15 = 64; +static constexpr int BYTES_PER_SMEM_15 = TILE_M_15 * TILE_N_15 * BYTES_PER_ELEMENT_15; +static constexpr int WARPS_M_15 = 1; +static constexpr int WARPS_N_15 = 1; +static constexpr int WARP_TILES_M_15 = TILE_M_15 / WARP_TILE_M_15; +static constexpr int WARP_TILES_N_15 = TILE_N_15 / WARP_TILE_N_15; +static constexpr int THREADS_PER_WARP_TILE_M_15 = 8; +static constexpr int THREADS_PER_WARP_TILE_N_15 = 4; +static constexpr int REGS_M_15 = WARP_TILES_M_15 / WARPS_M_15; +static constexpr int REGS_N_15 = WARP_TILES_N_15 / WARPS_N_15; +static constexpr int BYTES_PER_LD_15 = 128; +static constexpr int SWIZZLE_SCALE_15 = FORT_MIN(BYTES_PER_LD_15 / 16, WARPS_PER_GROUP_0 * 4); + +//global_load_shared_store_op 16 types +static constexpr int BITS_PER_ELEMENT_16 = 16; +static constexpr int BYTES_PER_ELEMENT_16 = BITS_PER_ELEMENT_16 / 8; +static constexpr int BYTES_PER_ACCESS_16 = 16; +static constexpr int TILE_M_16 = 128; +static constexpr int TILE_N_16 = 64; +static constexpr int BYTES_PER_SMEM_16 = TILE_M_16 * TILE_N_16 * BYTES_PER_ELEMENT_16; + +//mma_op 17 types +static constexpr int BYTES_PER_ELEMENT_17 = 2; +static constexpr int BYTES_PER_ACC_17 = 4; +static constexpr int WARP_TILE_M_17 = 16; +static constexpr int WARP_TILE_N_17 = 16; +static constexpr int WARP_TILE_K_17 = 16; +static constexpr int TILE_M_17 = 64; +static constexpr int TILE_N_17 = 64; +static constexpr int TILE_K_17 = 128; +static constexpr int WARPS_M_17 = 4; +static constexpr int WARPS_N_17 = 1; +static constexpr int WARP_TILES_M_17 = TILE_M_17 / WARP_TILE_M_17; +static constexpr int WARP_TILES_N_17 = TILE_N_17 / WARP_TILE_N_17; +static constexpr int MMA_STEPS_K_17 = TILE_K_17 / WARP_TILE_K_17; +static constexpr int WARP_REGS_17 = 8; +static constexpr int REGS_M_17 = WARP_TILES_M_17 / WARPS_M_17; +static constexpr int REGS_N_17 = WARP_TILES_N_17 / WARPS_N_17; + +// global_store_op 20 types + +// mma_loop_op 2 types +inline __device__ int2 compute_kv_loop_bounds(const int row_coord, const int ROW_TILE_SIZE, const int COL_TILE_SIZE, const int actual_seqlen_kv, const int actual_seqlen_q, const int shift_right_bound, const int left_bound) +{ + constexpr bool is_right_bound = true; + constexpr bool is_right_bound_bottom_right_alignment = false; + constexpr bool is_shift_right_bound = false; + constexpr bool is_left_bound = false; + constexpr bool is_left_bound_bottom_right_alignment = false; + + const int right_bound_diagonal = is_right_bound_bottom_right_alignment ? row_coord + (actual_seqlen_kv - actual_seqlen_q) : row_coord; + const int shifted_right_bound_diagonal = is_shift_right_bound ? right_bound_diagonal + shift_right_bound : right_bound_diagonal; + const int left_bound_diagonal = is_left_bound_bottom_right_alignment ? row_coord + (actual_seqlen_kv - actual_seqlen_q) : row_coord; + + const int kv_loop_left_bound = is_left_bound ? FORT_MAX(0, (left_bound_diagonal - left_bound) / COL_TILE_SIZE) : 0; + const int kv_loop_right_bound = is_right_bound ? FORT_MIN(FORT_DIV_UP(shifted_right_bound_diagonal + ROW_TILE_SIZE, COL_TILE_SIZE), FORT_DIV_UP(actual_seqlen_kv, COL_TILE_SIZE)) : FORT_DIV_UP(actual_seqlen_kv, COL_TILE_SIZE); + return make_int2(kv_loop_left_bound, kv_loop_right_bound); +} + +// output_loop_op 3 types +static constexpr int BYTES_PER_BANK_3 = 16; +static constexpr int LDS_TILE_M_3 = 16; +static constexpr int LDS_TILE_N_3 = 32; +static constexpr int ACC_CORE_MATRIX_ROWS_3 = 8; // NOTE: fixed due to GMMA design +static constexpr int ACC_CORE_MATRIX_COLS_3 = 8; // NOTE: fixed due to GMMA design +static constexpr int ELEMENTS_PER_VECTOR_3 = 8; +static constexpr int LDS_PER_TILE_3 = ELEMENTS_PER_VECTOR_3 / 2; +static constexpr int LDS_TILES_N_3 = FORT_MAX(CTA_TILE_N_1 / LDS_TILE_N_3, 1); +static constexpr int LDS_TILES_M_3 = FORT_MAX(1, (CTA_TILE_M_1 / GROUPS_M_1) / (LDS_TILE_M_3 * WARPS_PER_GROUP_0)); +static constexpr int VECTORS_PER_LDS_TILES_3 = LDS_TILE_M_3 / ACC_CORE_MATRIX_ROWS_3; +static constexpr int STS_PER_OUTPUT_TILE_N_3 = CTA_TILE_N_1 / ACC_CORE_MATRIX_COLS_3; +static constexpr int STSM_X4_PER_OUTPUT_TILE_N_3 = CTA_TILE_N_1 / ACC_CORE_MATRIX_COLS_3 / 2; +static constexpr int BANKS_PER_PAD_LINE_3 = BYTES_PER_ACC_1 == 4 ? 1 : 2; + +static constexpr int PADDING_BYTES_3 = BANKS_PER_PAD_LINE_3 * BYTES_PER_BANK_3 * LDS_PER_TILE_3; +)KERNEL" + R"KERNEL( +static constexpr int BYTES_PER_STS_PER_WARP_3= 16 * THREADS_PER_WARP_0; // 16 bytes due to STS_128 +static constexpr int BYTES_PER_LDS_TILE_3 = LDS_TILE_N_3 * LDS_TILE_M_3 * BYTES_PER_ACC_1 + PADDING_BYTES_3; +static constexpr int BYTES_PER_WARP_3 = LDS_TILES_N_3 * BYTES_PER_LDS_TILE_3; + +static constexpr int ELEMS_PER_STS_BLOCK_3 = 16; + +static constexpr int STG_THREADS_PER_TILE_N_3 = LDS_TILE_N_3 / ELEMENTS_PER_VECTOR_3; +static constexpr int EPILOGUE_SMEM_SIZE_PER_XMMA_M_3 = BYTES_PER_WARP_3 * WARPS_PER_GROUP_0; + +// shared_store_op 18 types + +// shared_load_op 19 types +static constexpr int BITS_PER_ELEMENT_19 = 32 * 1; +static constexpr int BYTES_PER_ELEMENT_19 = BITS_PER_ELEMENT_19 / 8; +static constexpr int BITS_PER_VECTOR_19 = BITS_PER_ELEMENT_19 * ELEMENTS_PER_VECTOR_3; +static constexpr int BYTES_PER_VECTOR_19 = BITS_PER_VECTOR_19 / 8; +static constexpr int REGISTERS_PER_VECTOR_19 = BITS_PER_VECTOR_19 / BITS_PER_REGISTER_0; + +//receive_op 0 code + +// === Section: Kernel Function === +)KERNEL" + R"KERNEL( +// === Section: Kernel Function === +extern "C" __global__ __launch_bounds__(384, 1) +void cudnn_generated_oss_sdpa_sm90_flash_fprop_wgmma_f16_knob_7_64x128x64_4x1x1_cga1x1x1_kernel0_0(const AttentionDescriptor_t attnDesc +, int num_tiles_1 +, const FastDivisor_t tiles_hr_div_1 +, const FastDivisor_t tiles_r_div_1 +, int32_t* tile_id_counter +, __grid_constant__ const CUtensorMap tma_Q +, __grid_constant__ const CUtensorMap tma_K +, float attn_scale +, float neg_infinity +, void* d_max +, fort::tensor_descriptor desc_max +, void* d_sum_exp +, fort::tensor_descriptor desc_sum_exp +, __grid_constant__ const CUtensorMap tma_V +, __grid_constant__ const CUtensorMap tma_O +) { + extern __shared__ char smem_[]; + uint32_t smem_0 = get_smem_pointer(smem_); + + //receive_op 0 decls + + // mma_pipeline_op 1 decls + const uint32_t tid = threadIdx.x % THREADS_PER_GROUP_0; + const uint32_t gid = threadIdx.x / THREADS_PER_GROUP_0; + const uint32_t wid = threadIdx.x / 32; + const uint32_t tiw = threadIdx.x % 32; + + const uint32_t elect_one = fort::elect_one_sync(); + + uint32_t smem_1 = smem_0; + uint32_t smem_bar_qkv_1 = smem_1; + uint32_t smem_page = smem_1 + 1024; + uint32_t smem_Q = smem_1 + SMEM_OFFSET_Q_1; + uint32_t smem_K = smem_1 + SMEM_OFFSET_K_1; + uint32_t smem_V = smem_1 + SMEM_OFFSET_V_1; + uint32_t smem_O = smem_1 + SMEM_OFFSET_O_1; + uint32_t smem_d_1 = smem_1 + SMEM_OFFSET_D_1; + uint32_t smem_band_bias_1 = smem_1 + SMEM_OFFSET_BAND_BIAS_1; + + uint32_t smem_bar_qkv_tma_1 = smem_bar_qkv_1; + uint32_t smem_bar_qkv_mma_1 = smem_bar_qkv_1 + 1 * 8 * (BUFFERS_Q_1 + BUFFERS_K_1 + BUFFERS_V_1 + BUFFERS_D_1 + 0); + uint32_t smem_tile_id_1 = smem_bar_qkv_1 + 2 * 8 * (BUFFERS_Q_1 + BUFFERS_K_1 + BUFFERS_V_1 + BUFFERS_D_1 + 0); + uint32_t smem_bar_tile_id_tma_1 = smem_tile_id_1 + 16; + uint32_t smem_bar_bias_tma_1 = smem_bar_tile_id_tma_1 + 8; + + uint32_t smem_bar_bias_mma_1 = smem_bar_bias_tma_1 + 8 * 2; + + if (elect_one) { + for (int i = wid; i < (BUFFERS_Q_1 + BUFFERS_K_1 + BUFFERS_V_1 + BUFFERS_D_1 + 0); i+=12) { + smem_bar_init(smem_bar_qkv_tma_1 + i * 8, 1); + smem_bar_init(smem_bar_qkv_mma_1 + i * 8, 2); + } + } + __syncthreads(); + + // mma_pipeline_op 1 code + const int actual_num_tiles_1 = num_tiles_1; + if (gid == 0) { + reg_dealloc<40>(); + const uint32_t local_wid = wid % 4; + + uint32_t tile_id; + uint32_t batch_coord_1 = 0, head_coord_1 = 0, q_row_coord = 0, rows_temp; + uint32_t head_coord_from_grid_1, head_coord_k_1, head_coord_v_1, head_coord_q_offset_1; + int actual_seqlen_kv_1 = attnDesc.s_kv, actual_seqlen_q_1 = attnDesc.s_q, oob_for_stats_1 = attnDesc.s_q; + + if (local_wid == 0) { + uint32_t smem_bar_q_tma = smem_bar_qkv_tma_1; + uint32_t smem_bar_k_tma = smem_bar_q_tma + 8 * BUFFERS_Q_1; + uint32_t smem_bar_v_tma = smem_bar_k_tma + 8 * BUFFERS_K_1; + + uint32_t smem_bar_q_mma = smem_bar_qkv_mma_1; + uint32_t smem_bar_k_mma = smem_bar_q_mma + 8 * BUFFERS_Q_1; + uint32_t smem_bar_v_mma = smem_bar_k_mma + 8 * BUFFERS_K_1; + + uint32_t buffer_id_q = 0; + uint32_t buffer_id_kv = 0; + uint32_t cnt_q = 0; + uint32_t cnt_kv = 0; + + uint32_t sts_page_buffer_id = 0; + uint32_t lds_page_buffer_id = 0; + + int cumulative_tiles = 0; + if (tiw == 0) { + tile_id = atomicAdd(tile_id_counter, 1); + } + tile_id = __shfl_sync(0xffffffff, tile_id, 0); + + named_barrier_wait(TILE_ID_SYNC_2_BARRIER_1, THREADS_ON_TILE_BARRIER_1); + if (tiw == 0) { + sts_32(smem_tile_id_1, &tile_id); + } + named_barrier_arrive(TILE_ID_SYNC_3_BARRIER_1, THREADS_ON_TILE_BARRIER_1); + + // Persistent loop over output tiles +#pragma unroll 1 + while (tile_id < actual_num_tiles_1) { + fastDivMod(tiles_hr_div_1, tile_id, batch_coord_1, rows_temp); + fastDivMod(tiles_r_div_1, rows_temp, head_coord_from_grid_1, q_row_coord); + q_row_coord *= TILE_M_1 * 2; + head_coord_1 = head_coord_from_grid_1; + head_coord_k_1 = head_coord_1 / attnDesc.q_heads_per_k; + head_coord_v_1 = head_coord_1 / attnDesc.q_heads_per_v; + + const int left_bound = 0; + const int shift_right_bound = 0; + + int2 kv_loop_bounds = compute_kv_loop_bounds(q_row_coord, TILE_M_1 * 2, TILE_N_1, actual_seqlen_kv_1, actual_seqlen_q_1, shift_right_bound, left_bound); + int kv_loop_left_bound = kv_loop_bounds.x; + int kv_loop_right_bound = kv_loop_bounds.y; + + //global_load_shared_store_op 4 decls + + //global_load_shared_store_op 7 decls + + //global_load_shared_store_op 16 decls + + uint32_t local_smem_q = smem_Q + buffer_id_q * SMEM_Q_1; + uint32_t local_smem_bar_tma_q = smem_bar_q_tma + buffer_id_q * 8; + uint32_t local_smem_bar_mma_q = smem_bar_q_mma + buffer_id_q * 8; + + if (kv_loop_left_bound < kv_loop_right_bound) { + uint32_t smem_bar_phase_q = cnt_q < BUFFERS_Q_1 ? 1 : 0; + + cnt_q = cnt_q < (2 * BUFFERS_Q_1 - 1) ? (cnt_q + 1) : 0; + buffer_id_q = buffer_id_q < ( BUFFERS_Q_1 - 1) ? (buffer_id_q + 1) : 0; + + if (!smem_bar_peek(local_smem_bar_mma_q, smem_bar_phase_q)) { + smem_bar_wait(local_smem_bar_mma_q, smem_bar_phase_q); + } + + smem_bar_set_transaction_count(local_smem_bar_tma_q, SMEM_Q_1, elect_one); + { + const uint32_t row_coord = q_row_coord; + + //global_load_shared_store_op 4 code +#pragma unroll + for (int i = 0; i < TILE_K_1 * BYTES_PER_ELEMENT_1; i += 128) { + utmaldg_4d_tiled(&tma_Q, + local_smem_q + i * TILE_M_1 * 2, + local_smem_bar_tma_q, + i / BYTES_PER_ELEMENT_1, + row_coord, + head_coord_1 + 0, + batch_coord_1, + elect_one); + } + } + } + +#pragma unroll 1 + for (int kv_loop = kv_loop_left_bound; kv_loop < kv_loop_right_bound; kv_loop++) { + const int p_col_coord = kv_loop * TILE_N_1; + + uint32_t local_smem_k = smem_K + buffer_id_kv * SMEM_K_1; + uint32_t local_smem_bar_tma_k = smem_bar_k_tma + buffer_id_kv * 8; + uint32_t local_smem_bar_mma_k = smem_bar_k_mma + buffer_id_kv * 8; + + uint32_t local_smem_v = smem_V + buffer_id_kv * SMEM_V_1; + uint32_t local_smem_bar_tma_v = smem_bar_v_tma + buffer_id_kv * 8; + + uint32_t local_smem_bar_mma_v = smem_bar_v_mma + buffer_id_kv * 8; + + uint32_t smem_bar_phase_kv = cnt_kv < BUFFERS_K_1 ? 1 : 0; + + cnt_kv = cnt_kv < (2 * BUFFERS_K_1 - 1) ? (cnt_kv + 1) : 0; + buffer_id_kv = buffer_id_kv < ( BUFFERS_K_1 - 1) ? (buffer_id_kv + 1) : 0; + + uint32_t kv_col_base = (kv_loop + 1) * TILE_N_1; + + if (!smem_bar_peek(local_smem_bar_mma_k, smem_bar_phase_kv)) { + smem_bar_wait(local_smem_bar_mma_k, smem_bar_phase_kv); + } + smem_bar_set_transaction_count(local_smem_bar_tma_k, SMEM_K_1, elect_one); + + //global_load_shared_store_op 7 code +#pragma unroll + for (int i = 0; i < TILE_K_1 * BYTES_PER_ELEMENT_1; i += 128) { + utmaldg_4d_tiled(&tma_K, + local_smem_k + i * TILE_N_1, + local_smem_bar_tma_k, + i / BYTES_PER_ELEMENT_1, + p_col_coord, + head_coord_k_1, + batch_coord_1, + elect_one); + } + if (!smem_bar_peek(local_smem_bar_mma_v, smem_bar_phase_kv)) { + smem_bar_wait(local_smem_bar_mma_v, smem_bar_phase_kv); + } + smem_bar_set_transaction_count(local_smem_bar_tma_v, SMEM_V_1, elect_one); + + //global_load_shared_store_op 16 code +#pragma unroll + for (int i = 0; i < TILE_K_1 * BYTES_PER_ELEMENT_1; i += 128 * CTA_MMA_1) { + utmaldg_4d_tiled(&tma_V, + local_smem_v + i * TILE_N_1, + local_smem_bar_tma_v, + 0 + i / BYTES_PER_ELEMENT_1, + p_col_coord, + head_coord_v_1, + batch_coord_1, + elect_one); + } + lds_page_buffer_id ^= 1; + } + if (kv_loop_left_bound < kv_loop_right_bound) { + lds_page_buffer_id ^= 1; + } + + if (tiw == 0) { + tile_id = atomicAdd(tile_id_counter, 1); + } + tile_id = __shfl_sync(0xffffffff, tile_id, 0); + + named_barrier_wait(TILE_ID_SYNC_2_BARRIER_1, THREADS_ON_TILE_BARRIER_1); + if (tiw == 0) { + sts_32(smem_tile_id_1, &tile_id); + } + named_barrier_arrive(TILE_ID_SYNC_3_BARRIER_1, THREADS_ON_TILE_BARRIER_1); + } + named_barrier_wait(TILE_ID_SYNC_2_BARRIER_1, THREADS_ON_TILE_BARRIER_1); + } + reg_dealloc<40>(); + } else { + reg_alloc<232>(); + named_barrier_arrive(TILE_ID_SYNC_2_BARRIER_1, THREADS_ON_TILE_BARRIER_1); + + const uint32_t local_wid = wid % 4; + const uint32_t local_gid = gid % 2; + + uint32_t tile_id; + uint32_t batch_coord_1, head_coord_1, q_row_coord, rows_temp; + uint32_t head_coord_from_grid_1, head_coord_k_1, head_coord_v_1, head_coord_q_offset_1; + int actual_seqlen_kv_1 = attnDesc.s_kv, actual_seqlen_q_1 = attnDesc.s_q, oob_for_stats_1 = attnDesc.s_q; + + uint32_t smem_bar_q_tma = smem_bar_qkv_tma_1; + uint32_t smem_bar_k_tma = smem_bar_q_tma + 8 * BUFFERS_Q_1; + uint32_t smem_bar_v_tma = smem_bar_k_tma + 8 * BUFFERS_K_1; + + uint32_t smem_bar_q_mma = smem_bar_qkv_mma_1; + uint32_t smem_bar_k_mma = smem_bar_q_mma + 8 * BUFFERS_Q_1; + uint32_t smem_bar_v_mma = smem_bar_k_mma + 8 * BUFFERS_K_1; + + uint32_t buffer_id_q = 0; + uint32_t buffer_id_kv = 0; + uint32_t cnt_q = 0; + uint32_t cnt_kv = 0; + + static constexpr float inv_ln2 = 1.4426950408889634074f; + static constexpr float ln2 = 0.6931471805599453094f; + + Gmma_descriptor gmma_desc_q(create_gmma_desc_q_1()); + Gmma_descriptor gmma_desc_k(create_gmma_desc_k_1()); + Gmma_descriptor gmma_desc_v(create_gmma_desc_v_1()); + + gmma_desc_q.set_smem(smem_Q + local_gid * TILE_M_1 * 128); + gmma_desc_k.set_smem(smem_K); + gmma_desc_v.set_smem(smem_V); + + named_barrier_wait(TILE_ID_SYNC_3_BARRIER_1, THREADS_ON_TILE_BARRIER_1); + if (tiw == 0) { + lds_32(&tile_id, smem_tile_id_1); + } + named_barrier_arrive(TILE_ID_SYNC_2_BARRIER_1, THREADS_ON_TILE_BARRIER_1); + tile_id = __shfl_sync(0xffffffff, tile_id, 0); + + // Persistent loop over output tiles +#pragma unroll 1 + while (tile_id < actual_num_tiles_1) { + fastDivMod(tiles_hr_div_1, tile_id, batch_coord_1, rows_temp); + fastDivMod(tiles_r_div_1, rows_temp, head_coord_from_grid_1, q_row_coord); + q_row_coord = q_row_coord * TILE_M_1 * 2; + head_coord_1 = head_coord_from_grid_1; + head_coord_k_1 = head_coord_1 / attnDesc.q_heads_per_k; + head_coord_v_1 = head_coord_1 / attnDesc.q_heads_per_v; + const int left_bound = 0; + const int shift_right_bound = 0; + + int2 kv_loop_bounds = compute_kv_loop_bounds(q_row_coord, TILE_M_1 * 2, TILE_N_1, actual_seqlen_kv_1, actual_seqlen_q_1, shift_right_bound, left_bound); + int kv_loop_left_bound = kv_loop_bounds.x; + int kv_loop_right_bound = kv_loop_bounds.y; + q_row_coord += local_gid * TILE_M_1; + + const int causal_mask_col = (tiw % 4) * 2; + const int causal_mask_row = q_row_coord + local_wid * 16 + (tiw / 4); + + // mma_loop_op 2 decls + + //global_load_shared_store_op 4 decls + + // shared_load_op 5 decls + + // shared_load_op 6 decls + + //global_load_shared_store_op 7 decls + + //mma_op 8 decls + r32 reg_8_0[REGS_M_8][REGS_N_8][WARP_REGS_8]; + + //pointwise_calc_op 9 decls + + //global_load_op 10 decls + r32 reg_10_0[1]; + reg_10_0[0] = reinterpret_cast(attn_scale); + const int oob_M_10 = 1; + const int oob_N_10 = 1; + + //mha_mask_op 11 decls + + //softmax_op 12 decls + r32 reg_12_0[REGS_M_12][REGS_N_12][4]; // output for second gemm + r32 reg_12_1[ROWS_PER_THREAD_12]; // acc_max + r32 reg_12_2[ROWS_PER_THREAD_12]; // p_max + r32 reg_12_3[ROWS_PER_THREAD_12]; // acc_sum + r32 reg_12_4[ROWS_PER_THREAD_12]; // p_sum + r32 reg_12_5[ROWS_PER_THREAD_12]; // softmax stats + float beta_12[ROWS_PER_THREAD_12]; + +#pragma unroll + for (int i = 0; i < ROWS_PER_THREAD_12; ++i) { + reinterpret_cast(reg_12_1[i]) = NEG_INFINITY; + reinterpret_cast(reg_12_3[i]) = 0.0f; + } + + //global_store_op 13 decls + int oob_M_13 = desc_max.dims[2]; + int row_13 = q_row_coord + local_wid * WARP_TILE_M_1 + (tiw / 4); + char *ptr_13 = reinterpret_cast(d_max) + + batch_coord_1 * desc_max.strides[0] * BYTES_PER_ELEMENT_13 + + head_coord_1 * desc_max.strides[1] * BYTES_PER_ELEMENT_13 + + row_13 * desc_max.strides[2] * BYTES_PER_ELEMENT_13; + + //global_store_op 14 decls + int oob_M_14 = desc_sum_exp.dims[2]; + int row_14 = q_row_coord + local_wid * WARP_TILE_M_1 + (tiw / 4); + char *ptr_14 = reinterpret_cast(d_sum_exp) + + batch_coord_1 * desc_sum_exp.strides[0] * BYTES_PER_ELEMENT_14 +)KERNEL" + R"KERNEL( + + head_coord_1 * desc_sum_exp.strides[1] * BYTES_PER_ELEMENT_14 + + row_14 * desc_sum_exp.strides[2] * BYTES_PER_ELEMENT_14; + + // shared_load_op 15 decls + + //global_load_shared_store_op 16 decls + + //mma_op 17 decls + + r32 reg_17_0[REGS_M_17][REGS_N_17][WARP_REGS_17]; + + //global_store_op 20 decls + static constexpr int cols_per_step_20 = (TILE_K_1 <= 128) ? TILE_K_1 : 64; + uint32_t smem_20 = smem_O + (local_gid * 4 + local_wid) * 16 * cols_per_step_20 * BYTES_PER_ELEMENT_1; + + const int swizzled_row_20 = (tiw % 16); + const int swizzled_col_20 = (tiw / 16); + constexpr int stsm_per_tile_20 = 128 / (BYTES_PER_BANK_1 * 2); + uint32_t stsm_base_20[stsm_per_tile_20]; + +#pragma unroll + for (int n = 0; n < stsm_per_tile_20; ++n) { + stsm_base_20[n] = smem_20 + ((swizzled_row_20 % 8) ^ (n * 2 + swizzled_col_20)) * BYTES_PER_BANK_1 + swizzled_row_20 * 128; + } + + const int row_20 = q_row_coord + (local_wid % 4) * 16; + + uint32_t local_smem_bar_tma_q = smem_bar_q_tma + buffer_id_q * 8; + uint32_t local_smem_bar_mma_q = smem_bar_q_mma + buffer_id_q * 8; + + if (local_gid == 1) { + named_barrier_arrive(SOFTMAX_2_BARRIER_1, 256); + } + + if (kv_loop_left_bound < kv_loop_right_bound) { + uint32_t smem_bar_phase_q = cnt_q < BUFFERS_Q_1 ? 0 : 1; + + cnt_q = cnt_q < (2 * BUFFERS_Q_1 - 1) ? (cnt_q + 1) : 0; + buffer_id_q = buffer_id_q < ( BUFFERS_Q_1 - 1) ? (buffer_id_q + 1) : 0; + + if (!smem_bar_peek(local_smem_bar_tma_q, smem_bar_phase_q)) { + smem_bar_wait(local_smem_bar_tma_q, smem_bar_phase_q); + } + } + + r32 reg_1_0[REGS_M_1][REGS_O_1][WARP_REGS_1]; + memset(®_1_0[0][0][0], 0, sizeof(reg_1_0)); + + named_barrier_wait(MATH_WORKGROUP_1, 256); +#pragma unroll 1 + for (int kv_loop = kv_loop_left_bound; kv_loop < kv_loop_right_bound; kv_loop++) { + static constexpr int kMulPipeCount = REGS_M_1 * REGS_N_1 * WARP_REGS_1; + + static constexpr int kSubtractPipeCount = 6; + static constexpr int kConvertPipeCount = 4; + static constexpr int oBlockSize = 2; + + const int p_col_coord = kv_loop * TILE_N_1; + const int causal_mask_col = p_col_coord + (tiw % 4) * 2; + const int causal_mask_row = q_row_coord + local_wid * 16 + (tiw / 4); + + uint32_t local_smem_bar_tma_k = smem_bar_k_tma + buffer_id_kv * 8; + uint32_t local_smem_bar_mma_k = smem_bar_k_mma + buffer_id_kv * 8; + + uint32_t local_smem_bar_tma_v = smem_bar_v_tma + buffer_id_kv * 8; + uint32_t local_smem_bar_mma_v = smem_bar_v_mma + buffer_id_kv * 8; + + uint32_t smem_bar_phase_kv = cnt_kv < BUFFERS_K_1 ? 0 : 1; + + cnt_kv = cnt_kv < (2 * BUFFERS_K_1 - 1) ? (cnt_kv + 1) : 0; + buffer_id_kv = buffer_id_kv < ( BUFFERS_K_1 - 1) ? (buffer_id_kv + 1) : 0; + + memset(®_8_0[0][0][0], 0, sizeof(reg_8_0)); + + if (!smem_bar_peek(local_smem_bar_tma_k, smem_bar_phase_kv)) { + smem_bar_wait(local_smem_bar_tma_k, smem_bar_phase_kv); + } + + warpgroup_arrive(); +#pragma unroll + for (int k = 0; k < REGS_K_1; ++k) { + if (k == REGS_K_1 - 1) { + BMM_S_GMMA_ISB(gmma_desc_q.desc, gmma_desc_k.desc, reinterpret_cast(reg_8_0[0])); + } else { + BMM_S_GMMA(gmma_desc_q.desc, gmma_desc_k.desc, reinterpret_cast(reg_8_0[0])); + } + int2 &tmp_desc_q = reinterpret_cast(gmma_desc_q.desc); + tmp_desc_q.x += BYTES_PER_GMMA_K_NO_4LSB_1; + if ((k % 4) == 3) { + tmp_desc_q.x -= BYTES_PER_GMMA_K_NO_4LSB_1 * 4; + tmp_desc_q.x += ((128*TILE_M_1*2) >> 4); + } + int2 &tmp_desc_k = reinterpret_cast(gmma_desc_k.desc); + tmp_desc_k.x += BYTES_PER_GMMA_K_NO_4LSB_1; + if ((k % 4) == 3) { + tmp_desc_k.x -= BYTES_PER_GMMA_K_NO_4LSB_1 * 4; + tmp_desc_k.x += ((128*TILE_N_1) >> 4); + } + } + { + int2 &tmp_desc_q = reinterpret_cast(gmma_desc_q.desc); + tmp_desc_q.x -= ((128 * TILE_M_1 * 2) >> 4) * (REGS_K_1 / 4); + int2 &tmp_desc_k = reinterpret_cast(gmma_desc_k.desc); + tmp_desc_k.x -= ((128 * TILE_N_1) >> 4) * (REGS_K_1 / 4); + } + gmma_desc_k.increment_smem_buffer(); + warpgroup_wait<0>(); + + if (local_wid == 0 && elect_one) { + smem_bar_arrive(local_smem_bar_mma_k); + } + + uint32_t wait_v_done = smem_bar_peek(local_smem_bar_tma_v, smem_bar_phase_kv); + + //mha_mask_op 11 code + if (kv_loop >= kv_loop_right_bound - 1) + { +#pragma unroll + for (int m = 0; m < REGS_M_11; ++m) { +#pragma unroll + for (int n = 0; n < REGS_N_11; ++n) { +#pragma unroll + for (int i = 0; i < WARP_REGS_11; ++i) { + int row = causal_mask_row + m * WARPS_PER_GROUP_0 * WARP_TILE_M_11 + ((i / 2) % 2) * 8; + int col = causal_mask_col + n * WARP_TILE_N_11 + i % 2 + (i / 4) * 8; + bool mask = compute_diagonal_band_mask_11(row, col, actual_seqlen_kv_1, actual_seqlen_q_1, shift_right_bound, left_bound); + if (!mask) { + reinterpret_cast(reg_8_0[m][n][i]) = neg_infinity; + } + } + } + } + } + + //global_load_op 10 code + + //mha_mask_op 11 code + if (kv_loop >= kv_loop_right_bound - 1) + { +#pragma unroll + for (int m = 0; m < REGS_M_11; ++m) { +#pragma unroll + for (int n = 0; n < REGS_N_11; ++n) { +#pragma unroll + for (int i = 0; i < WARP_REGS_11; ++i) { + int row = causal_mask_row + m * WARPS_PER_GROUP_0 * WARP_TILE_M_11 + ((i / 2) % 2) * 8; + int col = causal_mask_col + n * WARP_TILE_N_11 + i % 2 + (i / 4) * 8; + bool mask = compute_diagonal_band_mask_11(row, col, actual_seqlen_kv_1, actual_seqlen_q_1, shift_right_bound, left_bound); + if (!mask) { + reinterpret_cast(reg_8_0[m][n][i]) = neg_infinity; + } + } + } + } + } + + //softmax_op 12 code + static constexpr float inv_ln2_12 = 1.4426950408889634074f; + + float scaled_inv_ln2_12 = inv_ln2_12 * reinterpret_cast(reg_10_0[0]); + + r32 *flattened_s = reinterpret_cast(reg_8_0); + r32 *flattened_s_fp16 = reinterpret_cast(reg_12_0); + r32 *flattened_o = reinterpret_cast(reg_1_0); + + float tmp_max_level_0[REGS_N_12][4]; + float tmp_max_level_1[REGS_N_12][2]; + float tmp_max_level_2[REGS_N_12 / 2][2]; +#pragma unroll + for (int m = 0; m < REGS_M_12; ++m) { +#pragma unroll + for (int n = 0; n < REGS_N_12; ++n) { +#pragma unroll + for (int i = 0; i < WARP_REGS_12; i+=2) { + tmp_max_level_0[n][i/2] = fmaxf(reinterpret_cast(reg_8_0[m][n][i+0]), + reinterpret_cast(reg_8_0[m][n][i+1])); + } + } + } + // FMUL +#pragma unroll + for (int i = 0; i < kMulPipeCount; i++) { + reinterpret_cast(flattened_s[i]) = __fmul_rn(scaled_inv_ln2_12, reinterpret_cast(flattened_s[i])); + } + { +#pragma unroll + for (int n = 0; n < REGS_N_12; ++n) { +#pragma unroll + for (int i = 0; i < 2; i++) { + tmp_max_level_1[n][i] = fmaxf(tmp_max_level_0[n][i + 0], + tmp_max_level_0[n][i + 2]); + } + } + } + { +#pragma unroll + for (int n = 0; n < REGS_N_12; n+=2) { + tmp_max_level_2[n/2][0] = fmaxf(tmp_max_level_1[n][0], tmp_max_level_1[n+1][0]); + tmp_max_level_2[n/2][1] = fmaxf(tmp_max_level_1[n][1], tmp_max_level_1[n+1][1]); + } + } + if (TILE_N_1 == 256) { + float tmp_max_level_3[REGS_N_12 / 4][2]; + float tmp_max_level_4[FORT_MAX(1,REGS_N_12 / 8)][2]; +#pragma unroll + for (int n = 0; n < REGS_N_12; n+=4) { + tmp_max_level_3[n/4][0] = fmaxf(tmp_max_level_2[n/2][0], tmp_max_level_2[n/2+1][0]); + tmp_max_level_3[n/4][1] = fmaxf(tmp_max_level_2[n/2][1], tmp_max_level_2[n/2+1][1]); + } +#pragma unroll + for (int n = 0; n < REGS_N_12; n+=8) { + tmp_max_level_4[n/8][0] = fmaxf(tmp_max_level_3[n/4][0], tmp_max_level_3[n/4+1][0]); + tmp_max_level_4[n/8][1] = fmaxf(tmp_max_level_3[n/4][1], tmp_max_level_3[n/4+1][1]); + } + reinterpret_cast(reg_12_2[0]) = fmaxf(tmp_max_level_4[0][0], tmp_max_level_4[1][0]); + reinterpret_cast(reg_12_2[1]) = fmaxf(tmp_max_level_4[0][1], tmp_max_level_4[1][1]); + } else if (TILE_N_1 == 128) { + float tmp_max_level_3[REGS_N_12 / 4][2]; +#pragma unroll + for (int n = 0; n < REGS_N_12; n+=4) { + tmp_max_level_3[n/4][0] = fmaxf(tmp_max_level_2[n/2][0], tmp_max_level_2[n/2+1][0]); + tmp_max_level_3[n/4][1] = fmaxf(tmp_max_level_2[n/2][1], tmp_max_level_2[n/2+1][1]); + } + reinterpret_cast(reg_12_2[0]) = fmaxf(tmp_max_level_3[0][0], tmp_max_level_3[1][0]); + reinterpret_cast(reg_12_2[1]) = fmaxf(tmp_max_level_3[0][1], tmp_max_level_3[1][1]); + } else { + reinterpret_cast(reg_12_2[0]) = fmaxf(tmp_max_level_2[0][0], tmp_max_level_2[1][0]); + reinterpret_cast(reg_12_2[1]) = fmaxf(tmp_max_level_2[0][1], tmp_max_level_2[1][1]); + } + + // Apply the functor for each row inside each group of 4 threads. +#pragma unroll + for (int m = 0; m < ROWS_PER_THREAD_12; ++m) { + r32 tmp_0 = __shfl_xor_sync(uint32_t(-1), reg_12_2[m], 1); + + reinterpret_cast(reg_12_2[m]) = fmaxf(reinterpret_cast(reg_12_2[m]), + reinterpret_cast(tmp_0)); + + tmp_0 = __shfl_xor_sync(uint32_t(-1), reg_12_2[m], 2); + + reinterpret_cast(reg_12_2[m]) = fmaxf(reinterpret_cast(reg_12_2[m]), + reinterpret_cast(tmp_0)) * scaled_inv_ln2_12; + } + + // Update acc_max scale of flash attention +#pragma unroll + for (int m = 0; m < ROWS_PER_THREAD_12; ++m) { + r32 curr_max = reg_12_1[m]; + reinterpret_cast(reg_12_1[m]) = fmaxf(reinterpret_cast(reg_12_2[m]), + reinterpret_cast(curr_max)); + reg_12_2[m] = curr_max; + } + + float alpha_12[ROWS_PER_THREAD_12]; +#pragma unroll + for (int m = 0; m < ROWS_PER_THREAD_12; ++m) { + float p_max = reinterpret_cast(reg_12_2[m]); + float acc_max = reinterpret_cast(reg_12_1[m]); + + // disable FMA for scale * P - max + float tmp = (acc_max == NEG_INFINITY) ? p_max : __fsub_rn(p_max, acc_max); + reinterpret_cast(alpha_12[m]) = fp32_exp2(reinterpret_cast(tmp)); + } + +#pragma unroll + for (int i = 0; i < ROWS_PER_THREAD_12; ++i) { + reinterpret_cast(reg_12_4[i]) = 0.0f; + } + + // Broadcast scaled_max and alpha + float flattened_scaled_max[REGS_M_12 * REGS_N_12 * WARP_REGS_12]; + float flattened_alpha[REGS_M_12 * REGS_O_1 * WARP_REGS_12]; +#pragma unroll + for (int m = 0; m < REGS_M_12; ++m) { +#pragma unroll + for (int n = 0; n < REGS_N_12; ++n) { +#pragma unroll + for (int i = 0; i < WARP_REGS_12; ++i) { + int row_indx = m * 2 + (i / 2) % 2; + float max = reinterpret_cast(reg_12_1[row_indx]); + flattened_scaled_max[m * REGS_N_12 * WARP_REGS_12 + n * WARP_REGS_12 + i] = (max == NEG_INFINITY ? 0.0f : max); + } + } + } + +#pragma unroll + for (int m = 0; m < REGS_M_12; ++m) { +#pragma unroll + for (int n = 0; n < REGS_O_1; ++n) { +#pragma unroll + for (int i = 0; i < WARP_REGS_12; ++i) { + int row_indx = m * 2 + (i / 2) % 2; + flattened_alpha[m * REGS_O_1 * WARP_REGS_12 + n * WARP_REGS_12 + i] = alpha_12[row_indx]; + } + } + } + + cfence(); + if (local_gid == 1) { + named_barrier_wait(SOFTMAX_1_BARRIER_1, 256); + } else { + named_barrier_wait(SOFTMAX_2_BARRIER_1, 256); + } + cfence(); + + // FSUB +#pragma unroll + for (int i = 0; i < kSubtractPipeCount; i++) { +)KERNEL" + R"KERNEL( + reinterpret_cast(flattened_s[i]) = __fsub_rn(reinterpret_cast(flattened_s[i]), flattened_scaled_max[i]); + } + +#pragma unroll + + for (int i = 0; i < REGS_M_12 * REGS_N_12 * WARP_REGS_12; i+=2) { + int m_0 = (i-kConvertPipeCount+0) / (REGS_N_12 * WARP_REGS_12); + int ii_0 = (i-kConvertPipeCount+0) % WARP_REGS_12; + int m_1 = (i-kConvertPipeCount+1) / (REGS_N_12 * WARP_REGS_12); + int ii_1 = (i-kConvertPipeCount+1) % WARP_REGS_12; + int row_indx_0 = m_0 * 2 + (ii_0 / 2) % 2; + int row_indx_1 = m_1 * 2 + (ii_1 / 2) % 2; + // cast and FADD + if (i >= kConvertPipeCount) { + fp32x2_to_bf16x2(&flattened_s_fp16[(i - kConvertPipeCount)/2], &flattened_s[i - kConvertPipeCount]); + reinterpret_cast(reg_12_4[row_indx_0]) += reinterpret_cast(flattened_s[i-kConvertPipeCount+0]); + reinterpret_cast(reg_12_4[row_indx_1]) += reinterpret_cast(flattened_s[i-kConvertPipeCount+1]); + } + cfence(); + + // exp + flattened_s[i+0] = fp32_exp2(flattened_s[i+0]); + + // O update +#pragma unroll + for (int j = 0; j < oBlockSize; j++) { + if (oBlockSize*i+j < REGS_O_1 * REGS_M_1 * WARP_REGS_1) { + reinterpret_cast(flattened_o[oBlockSize*i+j]) *= flattened_alpha[oBlockSize*i+j]; + } + } + + // fmul + if (i + kMulPipeCount < REGS_M_12 * REGS_N_12 * WARP_REGS_12) { + reinterpret_cast(flattened_s[i+kMulPipeCount+0]) = __fmul_rn(scaled_inv_ln2_12, reinterpret_cast(flattened_s[i+kMulPipeCount+0])); + reinterpret_cast(flattened_s[i+kMulPipeCount+1]) = __fmul_rn(scaled_inv_ln2_12, reinterpret_cast(flattened_s[i+kMulPipeCount+1])); + } + // fsub + + if (i + kSubtractPipeCount < REGS_M_12 * REGS_N_12 * WARP_REGS_12) { + reinterpret_cast(flattened_s[i+kSubtractPipeCount+0]) = __fsub_rn(reinterpret_cast(flattened_s[i+kSubtractPipeCount+0]), flattened_scaled_max[i+kSubtractPipeCount+0]); + reinterpret_cast(flattened_s[i+kSubtractPipeCount+1]) = __fsub_rn(reinterpret_cast(flattened_s[i+kSubtractPipeCount+1]), flattened_scaled_max[i+kSubtractPipeCount+1]); + } + + // exp + flattened_s[i+1] = fp32_exp2(flattened_s[i+1]); + + // O update +#pragma unroll + for (int j = 0; j < oBlockSize; j++) { + if (oBlockSize+oBlockSize*i+j < REGS_O_1 * REGS_M_1 * WARP_REGS_1) { + reinterpret_cast(flattened_o[oBlockSize+oBlockSize*i+j]) *= flattened_alpha[oBlockSize+oBlockSize*i+j]; + } + } + } + if (local_gid == 0) { + named_barrier_arrive(SOFTMAX_1_BARRIER_1, 256); + } else { + named_barrier_arrive(SOFTMAX_2_BARRIER_1, 256); + } + cfence(); + +#pragma unroll + for (int i = REGS_M_12 * REGS_N_12 * WARP_REGS_12 - kConvertPipeCount; i < REGS_M_12 * REGS_N_12 * WARP_REGS_12; i+=2) { + int m_0 = (i + 0) / (REGS_N_12 * WARP_REGS_12); + int ii_0 = (i + 0) % WARP_REGS_12; + int m_1 = (i + 1) / (REGS_N_12 * WARP_REGS_12); + int ii_1 = (i + 1) % WARP_REGS_12; + int row_indx_0 = m_0 * 2 + (ii_0 / 2) % 2; + int row_indx_1 = m_1 * 2 + (ii_1 / 2) % 2; + fp32x2_to_bf16x2(&flattened_s_fp16[i/2], &flattened_s[i]); + reinterpret_cast(reg_12_4[row_indx_0]) += reinterpret_cast(flattened_s[i+0]); + reinterpret_cast(reg_12_4[row_indx_1]) += reinterpret_cast(flattened_s[i+1]); + } + + if (!wait_v_done) { + smem_bar_wait(local_smem_bar_tma_v, smem_bar_phase_kv); + } + + warpgroup_arrive(); +#pragma unroll + for (int k = 0; k < TILE_N_1 / GMMA_TILE_K_1; ++k) { + if (k == (TILE_N_1 / GMMA_TILE_K_1) - 1) { + BMM_O_GMMA_ISB(reg_12_0[0][k], gmma_desc_v.desc, reinterpret_cast(reg_1_0[0])); + } else { + BMM_O_GMMA(reg_12_0[0][k], gmma_desc_v.desc, reinterpret_cast(reg_1_0[0])); + } + int2 &tmp_desc_v = reinterpret_cast(gmma_desc_v.desc); + tmp_desc_v.x += BYTES_PER_GMMA_K_NO_4LSB_TRANS_1; + } + { + int2 &tmp_desc_v = reinterpret_cast(gmma_desc_v.desc); + tmp_desc_v.x -= BYTES_PER_GMMA_K_NO_4LSB_TRANS_1 * (TILE_N_1 / GMMA_TILE_K_1); + } + gmma_desc_v.increment_smem_buffer(); + + warpgroup_wait<0>(); + if (local_wid == 0 && elect_one) { + smem_bar_arrive(local_smem_bar_mma_v); + } + + // Update acc_sum of flash attention +#pragma unroll + for (int m = 0; m < ROWS_PER_THREAD_12; ++m) { + float p_sum = reinterpret_cast(reg_12_4[m]); + float acc_sum = reinterpret_cast(reg_12_3[m]); + + reinterpret_cast(reg_12_3[m]) = reinterpret_cast(alpha_12[m]) * acc_sum + p_sum; + } + } + if (kv_loop_left_bound < kv_loop_right_bound) { + gmma_desc_q.increment_smem_buffer(); + + if (local_wid == 0 && elect_one) { + smem_bar_arrive(local_smem_bar_mma_q); + } + } + if (local_gid == 0) { + named_barrier_arrive(SOFTMAX_2_BARRIER_1, 256); + } +#pragma unroll + for (int m = 0; m < ROWS_PER_THREAD_12; ++m) { + r32 tmp_0 = __shfl_xor_sync(uint32_t(-1), reg_12_3[m], 1); + + reinterpret_cast(reg_12_3[m]) = reinterpret_cast(reg_12_3[m]) + + reinterpret_cast(tmp_0); + + tmp_0 = __shfl_xor_sync(uint32_t(-1), reg_12_3[m], 2); + + reinterpret_cast(reg_12_3[m]) = reinterpret_cast(reg_12_3[m]) + + reinterpret_cast(tmp_0); + + float sum = reinterpret_cast(reg_12_3[m]); + reinterpret_cast(reg_12_1[m]) *= ln2; + + reinterpret_cast(reg_12_5[m]) = (sum == 0.f) ? NEG_INFINITY : reinterpret_cast(reg_12_1[m]) + __logf(sum); + + beta_12[m] = (sum == 0.f) ? 0.f : 1.f / sum; + } + +#pragma unroll + for (int m = 0; m < REGS_M_1; ++m) { +#pragma unroll + for (int n = 0; n < REGS_O_1; ++n) { +#pragma unroll + for (int i = 0; i < WARP_REGS_1; ++i) { + int row_indx = m * 2 + (i / 2) % 2; + + reinterpret_cast(reg_1_0[m][n][i]) *= beta_12[row_indx]; + } + } + } + + //global_store_op 13 code +#pragma unroll + for (int m = 0; m < ROWS_PER_THREAD_13; ++m) { + const int row_offset = (m % 2) * THREADS_PER_WARP_TILE_M_13 + (m / 2) * WARPS_M_13 * WARP_TILE_M_13; + + if (row_13 + row_offset < oob_M_13 && (tiw % 4) == 0) { + stg_32(ptr_13 + row_offset * desc_max.strides[2] * BYTES_PER_ELEMENT_13, ®_12_1[m]); + } + } + + //global_store_op 14 code +#pragma unroll + for (int m = 0; m < ROWS_PER_THREAD_14; ++m) { + const int row_offset = (m % 2) * THREADS_PER_WARP_TILE_M_14 + (m / 2) * WARPS_M_14 * WARP_TILE_M_14; + + if (row_14 + row_offset < oob_M_14 && (tiw % 4) == 0) { + stg_32(ptr_14 + row_offset * desc_sum_exp.strides[2] * BYTES_PER_ELEMENT_14, ®_12_3[m]); + } + } + + tmastg_wait(); + + //global_store_op 20 code +#pragma unroll + for (int s = 0; s < TILE_O_1 / 64; ++s) { +#pragma unroll + for (int n = 0; n < stsm_per_tile_20; ++n) { + uint32_t tmp[4]; + fp32x2_to_bf16x2(&tmp[0], ®_1_0[0][s * stsm_per_tile_20 + n][0]); + fp32x2_to_bf16x2(&tmp[1], ®_1_0[0][s * stsm_per_tile_20 + n][2]); + fp32x2_to_bf16x2(&tmp[2], ®_1_0[0][s * stsm_per_tile_20 + n][4]); + fp32x2_to_bf16x2(&tmp[3], ®_1_0[0][s * stsm_per_tile_20 + n][6]); + stsm_x4(stsm_base_20[n] + s * 64 * 16 * BYTES_PER_ELEMENT_1, tmp); + } + } + fence_view_async_shared(); + + int head_coord_loc = 0; + int row = row_20; + int head = head_coord_1; +#pragma unroll + for (int s = 0; s < TILE_K_1 / 64; ++s) { + utmastg_4d_tiled(&tma_O, + smem_20 + s * 64 * 16 * BYTES_PER_ELEMENT_1, + s * 64, + row, + head + head_coord_loc, + batch_coord_1); + } + tmastg_arrive(); + + named_barrier_wait(TILE_ID_SYNC_3_BARRIER_1, THREADS_ON_TILE_BARRIER_1); + if (tiw == 0) { + lds_32(&tile_id, smem_tile_id_1); + } + named_barrier_arrive(TILE_ID_SYNC_2_BARRIER_1, THREADS_ON_TILE_BARRIER_1); + tile_id = __shfl_sync(0xffffffff, tile_id, 0); + } + tmastg_wait(); + reg_alloc<232>(); + } +} +)KERNEL"; +inline constexpr size_t d64_fprop_source_len = sizeof(d64_fprop_source) - 1; + +inline constexpr const char d64_fprop_flags[] = R"FLAGS(--gpu-architecture=sm_90a +--std=c++17 +--define-macro=__x86_64__ +-w +--define-macro=__CUDACC_RTC__ +-default-device +--use_fast_math +)FLAGS"; +inline constexpr size_t d64_fprop_flags_len = sizeof(d64_fprop_flags) - 1; + +} // namespace generated +} // namespace experimental +} // namespace cudnn_frontend + +#if defined(_MSC_VER) +#pragma warning(pop) +#endif + +#if defined(__clang__) +#pragma clang diagnostic pop +#endif diff --git a/deps/cudnn-frontend/include/cudnn_frontend/graph_helpers.h b/deps/cudnn-frontend/include/cudnn_frontend/graph_helpers.h new file mode 100644 index 0000000..d2ce9e3 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/graph_helpers.h @@ -0,0 +1,587 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#pragma once + +#include +#include + +#include +#include +#include +#include +#include + +namespace cudnn_frontend { + +enum class [[nodiscard]] error_code_t { + OK, + ATTRIBUTE_NOT_SET, + SHAPE_DEDUCTION_FAILED, + INVALID_TENSOR_NAME, + INVALID_VARIANT_PACK, + GRAPH_NOT_SUPPORTED, + GRAPH_EXECUTION_PLAN_CREATION_FAILED, + GRAPH_EXECUTION_FAILED, + HEURISTIC_QUERY_FAILED, + UNSUPPORTED_GRAPH_FORMAT, + CUDA_API_FAILED, + CUDNN_BACKEND_API_FAILED, + INVALID_CUDA_DEVICE, + HANDLE_ERROR, + INVALID_VALUE, + NVRTC_COMPILATION_FAILED +}; + +typedef struct [[nodiscard]] error_object { + error_code_t code; + std::string err_msg; + error_object() : code(error_code_t::OK), err_msg("") {}; + error_object(error_code_t err, std::string msg) : code(err), err_msg(msg) {}; + + error_code_t + get_code() { + return code; + } + + std::string + get_message() { + return err_msg; + } + + bool + is_good() const { + return code == error_code_t::OK; + } + + bool + is_bad() const { + return !is_good(); + } + + bool + operator==(error_code_t compare_code) { + return code == compare_code; + } + + bool + operator!=(error_code_t compare_code) { + return code != compare_code; + } + +} error_t; + +#ifdef WIN32 +#define CUDNN_FRONTEND_WHILE_FALSE \ + __pragma(warning(push)) __pragma(warning(disable : 4127)) while (0) __pragma(warning(pop)) +#else +#define CUDNN_FRONTEND_WHILE_FALSE while (0) +#endif + +#define CHECK_CUDNN_FRONTEND_ERROR(x) \ + do { \ + if (auto retval = x; retval.is_bad()) { \ + CUDNN_FE_LOG_LABEL_ENDL("ERROR: " << #x << " at " << __FILE__ << ":" << __LINE__); \ + return retval; \ + } \ + } \ + CUDNN_FRONTEND_WHILE_FALSE + +#define RETURN_CUDNN_FRONTEND_ERROR_IF(cond, retval, message) \ + do { \ + if (cond) { \ + if (retval == error_code_t::OK) { \ + CUDNN_FE_LOG_LABEL("INFO: "); \ + } else { \ + CUDNN_FE_LOG_LABEL("ERROR: "); \ + } \ + CUDNN_FE_LOG(message << ". " << retval << " because (" << #cond ") at " << __FILE__ << ":" << __LINE__ \ + << "\n"); \ + return {retval, message}; \ + } \ + } \ + CUDNN_FRONTEND_WHILE_FALSE + +#define _CUDNN_CHECK_CUDNN_ERROR(x) \ + do { \ + if (auto cudnn_retval = x; cudnn_retval != CUDNN_STATUS_SUCCESS) { \ + std::stringstream error_msg; \ + error_msg << #x << " failed with message: " << detail::get_last_error_string_() \ + << ", and code: " << detail::get_error_string(cudnn_retval); \ + CUDNN_FE_LOG_LABEL_ENDL("ERROR: " << error_msg.str() << " at " << __FILE__ << ":" << __LINE__); \ + return {error_code_t::CUDNN_BACKEND_API_FAILED, error_msg.str()}; \ + } \ + } \ + CUDNN_FRONTEND_WHILE_FALSE + +#define _CUDNN_CHECK_CUDA_ERROR(x) \ + do { \ + if (auto cuda_retval = x; cuda_retval != cudaSuccess) { \ + std::stringstream error_msg; \ + error_msg << #x << " failed with " << detail::cuda_get_error_string(cuda_retval); \ + CUDNN_FE_LOG_LABEL_ENDL("ERROR: " << error_msg.str() << " at " << __FILE__ << ":" << __LINE__); \ + return {error_code_t::CUDA_API_FAILED, error_msg.str()}; \ + } \ + } \ + CUDNN_FRONTEND_WHILE_FALSE + +NLOHMANN_JSON_SERIALIZE_ENUM(error_code_t, + { + {error_code_t::OK, "OK"}, + {error_code_t::ATTRIBUTE_NOT_SET, "ATTRIBUTE_NOT_SET"}, + {error_code_t::SHAPE_DEDUCTION_FAILED, "SHAPE_DEDUCTION_FAILED"}, + {error_code_t::INVALID_TENSOR_NAME, "INVALID_TENSOR_NAME"}, + {error_code_t::INVALID_VARIANT_PACK, "INVALID_VARIANT_PACK"}, + {error_code_t::GRAPH_NOT_SUPPORTED, "GRAPH_NOT_SUPPORTED"}, + {error_code_t::GRAPH_EXECUTION_PLAN_CREATION_FAILED, + "GRAPH_EXECUTION_PLAN_CREATION_FAILED"}, + {error_code_t::GRAPH_EXECUTION_FAILED, "GRAPH_EXECUTION_FAILED"}, + {error_code_t::HEURISTIC_QUERY_FAILED, "HEURISTIC_QUERY_FAILED"}, + {error_code_t::CUDNN_BACKEND_API_FAILED, "CUDNN_BACKEND_API_FAILED"}, + {error_code_t::CUDA_API_FAILED, "CUDA_API_FAILED"}, + {error_code_t::INVALID_CUDA_DEVICE, "INVALID_CUDA_DEVICE"}, + {error_code_t::UNSUPPORTED_GRAPH_FORMAT, "UNSUPPORTED_GRAPH_FORMAT"}, + {error_code_t::HANDLE_ERROR, "HANDLE_ERROR"}, + {error_code_t::INVALID_VALUE, "INVALID_VALUE"}, + {error_code_t::NVRTC_COMPILATION_FAILED, "NVRTC_COMPILATION_FAILED"}, + }) + +static inline std::ostream& +operator<<(std::ostream& os, const error_code_t& mode) { +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + os << json{mode}; +#else + os << int(mode); +#endif + return os; +} + +static inline std::ostream& +operator<<(std::ostream& os, cudnn_frontend::error_object& err) { + os << err.get_code() << err.get_message(); + return os; +} + +static bool +allowAllConfig(cudnnBackendDescriptor_t engine_config) { + (void)engine_config; + return false; +} + +namespace detail { + +inline bool +is_activation_backward_mode(PointwiseMode_t const mode) { + return ((mode == PointwiseMode_t::RELU_BWD) || (mode == PointwiseMode_t::TANH_BWD) || + (mode == PointwiseMode_t::SIGMOID_BWD) || (mode == PointwiseMode_t::ELU_BWD) || + (mode == PointwiseMode_t::GELU_BWD) || (mode == PointwiseMode_t::GELU_APPROX_TANH_BWD) || + (mode == PointwiseMode_t::SOFTPLUS_BWD) || (mode == PointwiseMode_t::SWISH_BWD)); +} + +// Creates dense, non-overlapping strides from given dim and stride_order. +// For example, if a is a 4D tensor with dimensions labeled NCHW, then strided(a, (3, 0, 2, 1)) produces +// strides where the C dimension has a corresponding stride of one. +inline std::vector +generate_stride(std::vector const& dim, std::vector const& stride_order) { + size_t num_dims = dim.size(); + std::vector stride(num_dims); + + // Sort the dimensions according to strides from least to greatest. + // Example, dim = (2, 3, 4, 5) stride_order = (3, 1, 2, 0) + // sorted_stride_order = ((0, (3, 5)), (1, (1, 3)), (2, (2, 4)), (3, (0, 2))) + std::vector>> sorted_stride_order; + for (size_t i = 0; i < num_dims; ++i) { + sorted_stride_order.push_back({stride_order[i], {i, dim[i]}}); + } + std::sort(sorted_stride_order.begin(), sorted_stride_order.end()); + + // As dims have now been ordered starting from fastest changing, + // just fill in strides by iterating linearly over them. + int64_t product = 1; + for (size_t i = 0; i < num_dims; ++i) { + stride[sorted_stride_order[i].second.first] = product; + product *= sorted_stride_order[i].second.second; + } + + return stride; +} + +// Generate NHWC stride_order +inline std::vector +generate_NHWC_stride_order(int64_t const num_dims) { + std::vector stride_order(num_dims); + + int64_t order = 0; + stride_order[1] = order++; + for (size_t i = num_dims - 1; i > 1; --i) { + stride_order[i] = order++; + } + stride_order[0] = order; + + return stride_order; +} + +// Generate row major stride_order for matrices +// dim = (*, M, N) where * is batch dimsensions +// strides should be (..., N, 1) +inline std::vector +generate_row_major_stride_order(int64_t const num_dims) { + std::vector stride_order(num_dims); + + int64_t order = num_dims - 1; + std::generate(stride_order.begin(), stride_order.end(), [&order] { return order--; }); + + return stride_order; +} + +// Generate column major stride_order for matrices +// dim = (*, M, N) +// strides should be (*, 1, M) +inline std::vector +generate_column_major_stride_order(int64_t const num_dims) { + std::vector stride_order = generate_row_major_stride_order(num_dims); + if (num_dims > 2) { + std::swap(stride_order[num_dims - 1], stride_order[num_dims - 2]); + } + return stride_order; +} + +/** + * @brief Computes the common shape with the fewest dimensions that all input shapes can be broadcast to. + * + * This function takes a vector of shapes and calculates a common shape that all input shapes + * can be broadcast to. It follows broadcasting rules similar to those used in NumPy. + * + * @param _shapes A vector of vectors, where each inner vector represents a shape. + * Each shape is a sequence of dimension sizes. + * @param[out] common_shape The computed broadcast shape is stored in this vector. + * It will be cleared and resized as necessary. + * + * @return error_t An error code indicating the result of the operation + * + * @note + * - Shapes are processed from right to left (last dimension to first). + * - A dimension of size 1 can be broadcast to any size. + * - Non-1 dimensions must match exactly for broadcasting. + * - The resulting shape will have the maximum number of dimensions among all input shapes. + * + * @example + * std::vector> shapes = {{3, 1, 4}, {1, 2, 4}, {2, 4}}; + * std::vector result; + * error_t err = compute_broadcast_shape(shapes, result); + * // If err == error_code_t::OK, result will be {3, 2, 4} + */ +inline error_t +compute_broadcast_shape(const std::vector>& _shapes, std::vector& common_shape) { + // Filter out empty shapes + std::vector> shapes; + std::copy_if(_shapes.begin(), _shapes.end(), std::back_inserter(shapes), [](const std::vector& shape) { + return !shape.empty(); + }); + + // Short-circuits if there are no input shapes + RETURN_CUDNN_FRONTEND_ERROR_IF( + shapes.empty(), error_code_t::SHAPE_DEDUCTION_FAILED, "All input shapes provided are empty."); + + // Find the maximum dimension + int64_t max_dim = std::max_element(shapes.begin(), + shapes.end(), + [](const std::vector& a, const std::vector& b) { + return a.size() < b.size(); + }) + ->size(); + + // Initialize common_shape with 1s + common_shape.assign(max_dim, 1); + + for (const auto& shape : shapes) { + for (int idx = -1; idx >= -static_cast(shape.size()); --idx) { + int64_t common_idx = common_shape.size() + idx; + int64_t shape_idx = shape.size() + idx; + + if (common_shape[common_idx] == 1) { + common_shape[common_idx] = shape[shape_idx]; + } + + RETURN_CUDNN_FRONTEND_ERROR_IF((shape[shape_idx] != 1) && (common_shape[common_idx] != shape[shape_idx]), + error_code_t::SHAPE_DEDUCTION_FAILED, + "dimensions mismatch as broadcasting 2 non-one dimension sizes."); + } + } + + return {error_code_t::OK, ""}; +} +/** + * @brief Generates a stride order preserving the format of the input tensor. + * + * This function derives the exact stride order from the input tensor's strides. + * It returns the indices of the strides in ascending order of stride values. + * + * @param input_stride The stride of the input tensor + * @param output_dim_size The number of dimensions in the output tensor + * @return std::vector The generated stride order + */ +inline error_t +generate_stride_order_preserving_format(const std::vector& input_stride, + size_t output_dim_size, + std::vector& stride_order) { + std::vector indices(input_stride.size()); + std::iota(indices.begin(), indices.end(), 0); + + // Sort indices based on stride values in descending order + std::sort(indices.begin(), indices.end(), [&input_stride](int64_t i, int64_t j) { + return input_stride[i] < input_stride[j]; + }); + + // Enable this after further debug + // std::set stride_set(input_stride.begin(), input_stride.end()); + // RETURN_CUDNN_FRONTEND_ERROR_IF((stride_set.size() != input_stride.size()), + // error_code_t::SHAPE_DEDUCTION_FAILED, + // "Have multiple stride with same value. Cant determine stride order"); + + // Create the stride order + stride_order.resize(input_stride.size()); + for (size_t i = 0; i < indices.size(); ++i) { + stride_order[indices[i]] = i; + } + + // If output_dim_size is larger, pad with remaining dimensions + if (output_dim_size > input_stride.size()) { + size_t start = stride_order.size(); + stride_order.resize(output_dim_size); + std::iota(stride_order.begin() + start, stride_order.end(), start); + } + + return {error_code_t::OK, ""}; +} + +/** + * @brief Infers the output dimensions for a matrix multiplication operation. + * + * This function calculates the output dimensions of a matrix multiplication + * based on the input dimensions of tensors A and B. It uses compute_broadcast_shape + * for batch dimensions and ensures the last two dimensions are correct for matrix multiplication. + * + * @param a_dim Dimensions of the first input tensor (A). + * @param b_dim Dimensions of the second input tensor (B). + * @param output_dim Reference to the vector where the output dimensions will be stored. + * @return error_t An error code indicating the result of the operation. + */ +inline error_t +generate_matmul_output_dim(const std::vector& a_dim, + const std::vector& b_dim, + std::vector& output_dim) { + // Ensure a_dim and b_dim have at least 2 dimensions + if (a_dim.size() < 2 || b_dim.size() < 2) { + return {error_code_t::SHAPE_DEDUCTION_FAILED, "Input tensors must have at least 2 dimensions for matmul."}; + } + + // Check if inner dimensions are compatible + if (a_dim[a_dim.size() - 1] != b_dim[b_dim.size() - 2]) { + return {error_code_t::SHAPE_DEDUCTION_FAILED, + "Inner dimensions of input tensors are not compatible for matmul."}; + } + + // Prepare shapes for broadcasting + std::vector a_batch_dim(a_dim.begin(), a_dim.end() - 2); + std::vector b_batch_dim(b_dim.begin(), b_dim.end() - 2); + + // Compute broadcast shape for batch dimensions + std::vector broadcasted_batch; + CHECK_CUDNN_FRONTEND_ERROR(detail::compute_broadcast_shape({a_batch_dim, b_batch_dim}, broadcasted_batch)); + + // Construct final output shape + output_dim = broadcasted_batch; + output_dim.push_back(a_dim[a_dim.size() - 2]); // M from A + output_dim.push_back(b_dim[b_dim.size() - 1]); // N from B + + return {error_code_t::OK, ""}; +} + +inline std::string +to_hex(const void* data, size_t num_elements, size_t elem_size) { + const auto* bytes = static_cast(data); + std::stringstream ss; + ss << "["; + for (size_t i = 0; i < num_elements; ++i) { + if (i > 0) ss << ", "; + ss << "0x" << std::hex << std::uppercase; + switch (elem_size) { + case 1: + ss << static_cast(bytes[i]); + break; + case 2: + ss << *reinterpret_cast(&bytes[i * 2]); + break; + case 4: + ss << *reinterpret_cast(&bytes[i * 4]); + break; + case 8: + ss << *reinterpret_cast(&bytes[i * 8]); + break; + default: + ss << "?"; + } + } + ss << "]"; + return ss.str(); +} + +inline std::string +to_decimal(const void* data, size_t num_elements, size_t elem_size) { + const auto* bytes = static_cast(data); + std::stringstream ss; + ss << "["; + for (size_t i = 0; i < num_elements; ++i) { + if (i > 0) ss << ", "; + switch (elem_size) { + case 1: + ss << static_cast(bytes[i]); + break; + case 2: + ss << *reinterpret_cast(&bytes[i * 2]); + break; + case 4: + ss << *reinterpret_cast(&bytes[i * 4]); + break; + case 8: + ss << *reinterpret_cast(&bytes[i * 8]); + break; + default: + ss << "?"; + } + } + ss << "]"; + return ss.str(); +} + +inline std::string +to_base64(const void* data, size_t total_bytes) { + static const char table[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + const auto* bytes = static_cast(data); + std::string result; + result.reserve(((total_bytes + 2) / 3) * 4); + for (size_t i = 0; i < total_bytes; i += 3) { + uint32_t n = static_cast(bytes[i]) << 16; + if (i + 1 < total_bytes) n |= static_cast(bytes[i + 1]) << 8; + if (i + 2 < total_bytes) n |= static_cast(bytes[i + 2]); + result.push_back(table[(n >> 18) & 0x3F]); + result.push_back(table[(n >> 12) & 0x3F]); + result.push_back((i + 1 < total_bytes) ? table[(n >> 6) & 0x3F] : '='); + result.push_back((i + 2 < total_bytes) ? table[n & 0x3F] : '='); + } + return result; +} + +inline error_t +log_dump_tensor_content(int64_t uid, + std::string const& name, + void* ptr, + size_t num_elements, + size_t elem_size, + char fmt, + cudaStream_t stream) { + if (!isLoggingEnabled()) return {error_code_t::OK, ""}; + + size_t total_bytes = num_elements * elem_size; + + cudaPointerAttributes attr; + _CUDNN_CHECK_CUDA_ERROR(cuda_pointer_get_attributes(&attr, ptr)); + + std::vector host_buf(total_bytes); + if (attr.type == cudaMemoryTypeDevice || attr.type == cudaMemoryTypeManaged) { + _CUDNN_CHECK_CUDA_ERROR(cuda_mem_cpy_async(host_buf.data(), ptr, total_bytes, cudaMemcpyDeviceToHost, stream)); + _CUDNN_CHECK_CUDA_ERROR(cuda_stream_synchronize(stream)); + } else { + std::memcpy(host_buf.data(), ptr, total_bytes); + } + + std::string data_str; + switch (fmt) { + case 'x': + data_str = to_hex(host_buf.data(), num_elements, elem_size); + break; + case 'd': + data_str = to_decimal(host_buf.data(), num_elements, elem_size); + break; + case 'b': + data_str = to_base64(host_buf.data(), total_bytes); + break; + default: + data_str = to_hex(host_buf.data(), num_elements, elem_size); + } + CUDNN_FE_LOG_LABEL_ENDL("Tensor Dump Uid: " << uid << " Name: " << name << " Data: " << data_str); + return {error_code_t::OK, ""}; +} + +inline error_t +log_variant_pack_memory_type(int64_t uid, void* ptr) { + if (!isLoggingEnabled()) return {error_code_t::OK, ""}; + + cudaPointerAttributes attributes; + _CUDNN_CHECK_CUDA_ERROR(cuda_pointer_get_attributes(&attributes, ptr)); + + auto memory_type_to_string = [](cudaMemoryType type) { + switch (type) { + case cudaMemoryTypeHost: + return std::string("Host"); + case cudaMemoryTypeDevice: + return std::string("Device"); + case cudaMemoryTypeManaged: + return std::string("Managed"); + case cudaMemoryTypeUnregistered: + return std::string("Unregistered"); + default: + return "UNKNOWN cudaMemoryType (" + std::to_string(type) + ")"; + } + }; + + auto ptr_to_string = [](void* p) { + std::stringstream ss; + ss << "0x" << std::hex << std::setw(sizeof(void*) * 2) << std::setfill('0') << reinterpret_cast(p); + return ss.str(); + }; + + // clang-format off + CUDNN_FE_LOG_LABEL_ENDL("Variant Pack" << std::setw(0) << " Uid: " << std::setw(20) << uid + << std::setw(0) << " MemoryType: " << std::setw(12) << memory_type_to_string(attributes.type) + << std::setw(0) << " Device: " << std::setw(4) << attributes.device + << std::setw(0) << " UnifiedPtr: " << std::setw(20) << ptr_to_string(ptr) + << std::setw(0) << " DevicePtr: " << std::setw(20) << ptr_to_string(attributes.devicePointer) + << std::setw(0) << " HostPtr: " << std::setw(20) << ptr_to_string(attributes.hostPointer)); + // clang-format on + return {error_code_t::OK, ""}; +} + +} // namespace detail + +class cudnnGraphNotSupportedException : public std::runtime_error { + public: + cudnnGraphNotSupportedException(const char* message) throw() : std::runtime_error(message) {} + + virtual const char* + what() const throw() { + return std::runtime_error::what(); + } +}; + +} // namespace cudnn_frontend \ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/graph_interface.h b/deps/cudnn-frontend/include/cudnn_frontend/graph_interface.h new file mode 100644 index 0000000..7cac72b --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/graph_interface.h @@ -0,0 +1,3878 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "../cudnn_frontend_version.h" +#include "node/batchnorm.h" +#include "node/batchnorm_inference.h" +#include "node/bn_finalize.h" +#include "node/conv_fprop.h" +#include "node/conv_dgrad.h" +#include "node/conv_wgrad.h" +#include "node/dbn.h" +#include "node/dln.h" +#include "node/dbn_weight.h" +#include "node/genstats.h" +#include "node/layernorm.h" +#include "node/adaptive_layernorm.h" +#include "node/instancenorm.h" +#include "node/rmsnorm.h" +#include "node/rope.h" +#include "node/rope_backward.h" +#include "node/resample.h" +#include "node/reshape.h" +#include "node/slice.h" +#include "node/transpose.h" +// #include "node/scaled_dot_product_attention.h" +#include "node/scaled_dot_product_flash_attention.h" +#include "node/sdpa_fp8_bwd.h" +#include "node/block_scale_quantize.h" +#include "node/block_scale_dequantize.h" +#include "node/concatenate.h" +#include "node/moe_grouped_matmul.h" +#include "node/moe_grouped_matmul_bwd.h" + +#include "backend/backend_descriptor.h" +#include "plans.h" +#include "knobs.h" +#include "graph_helpers.h" +#include "backend/kernel_cache.h" + +namespace cudnn_frontend::graph { + +class Graph : public ICudnn, public INode { + private: + std::unordered_set> full_graph_inputs; + std::unordered_set used_uids; + int64_t fe_workspace_size = 0; + uint64_t graph_uid; + + std::unordered_set> deserialized_tensor_properties; + std::unordered_map deserialized_pass_by_value; + std::unordered_map>> deserialized_workspace_modifications; + + // Cached values computed during build/deserialize, used during execute to avoid repeated collection. + // These are mutable because execute() is const but needs non-const access for pointer extraction. + mutable std::unordered_map cached_pass_by_value; + mutable std::unordered_map>> cached_workspace_modifications; + + // char: 'x'=hex, 'd'=decimal, 'b'=base64 + std::vector, char>> tensors_to_dump; + + error_t + get_pre_assigned_uids(std::unordered_set &used_uids) { + for (auto const &input : full_graph_inputs) { + if (input->has_uid()) { + auto uid = input->get_uid(); + auto iter = used_uids.find(uid); + RETURN_CUDNN_FRONTEND_ERROR_IF(iter != used_uids.end(), + error_code_t::INVALID_VALUE, + "uid " + std::to_string(uid) + " for tensor named " + input->get_name() + + " has been already assigned to another tensor."); + used_uids.insert(uid); + } + } + for (auto const &output : full_graph_outputs) { + if (output->has_uid()) { + auto uid = output->get_uid(); + auto iter = used_uids.find(uid); + RETURN_CUDNN_FRONTEND_ERROR_IF(iter != used_uids.end(), + error_code_t::INVALID_VALUE, + "uid " + std::to_string(uid) + " for tensor named " + + output->get_name() + + " has been already assigned to another tensor."); + used_uids.insert(uid); + } + } + + return {error_code_t::OK, ""}; + } + + error_t + log_tensors_to_dump_(cudnnHandle_t handle, + std::unordered_map const &tensor_uid_to_pointer_map) const { + if (!isLoggingTensorDumpEnabled()) { + return {error_code_t::OK, ""}; + } + + for (auto const &[uid, ptr] : tensor_uid_to_pointer_map) { + CHECK_CUDNN_FRONTEND_ERROR(detail::log_variant_pack_memory_type(uid, ptr)); + } + + cudaStream_t stream; + _CUDNN_CHECK_CUDNN_ERROR(detail::get_stream(handle, &stream)); + for (auto const &[tensor, fmt] : tensors_to_dump) { + auto it = tensor_uid_to_pointer_map.find(tensor->get_uid()); + if (it != tensor_uid_to_pointer_map.end()) { + auto const &dims = tensor->get_dim(); + size_t num_elements = 1; + for (auto d : dims) num_elements *= static_cast(d); + size_t elem_size = detail::get_data_type_size(tensor->get_data_type()); + CHECK_CUDNN_FRONTEND_ERROR(detail::log_dump_tensor_content( + it->first, tensor->get_name(), it->second, num_elements, elem_size, fmt, stream)); + } + } + + return {error_code_t::OK, ""}; + } + + error_t + pre_validate_node() const override final { + RETURN_CUDNN_FRONTEND_ERROR_IF( + (context.get_dynamic_shape_enabled() || kernel_cache != nullptr) && detail::get_backend_version() < 90400, + error_code_t::GRAPH_NOT_SUPPORTED, + "Dynamic shapes or kernel caching enabled, but cuDNN version < 9.4!"); + RETURN_CUDNN_FRONTEND_ERROR_IF(((context.get_dynamic_shape_enabled() == false) && (kernel_cache != nullptr)), + error_code_t::GRAPH_NOT_SUPPORTED, + "Kernel caching enabled but dynamic shapes is disabled"); + if (detail::get_backend_version() != detail::get_compiled_version()) { + CUDNN_FE_LOG_LABEL_ENDL("INFO: The cuDNN version used at compilation (" + << detail::get_compiled_version() << ") and the one used at runtime (" + << detail::get_backend_version() << ") differ."); + } + return {error_code_t::OK, ""}; + } + + error_t + infer_properties_node() override final { + return {error_code_t::OK, ""}; + } + + error_t + post_validate_node() const override final { + return {error_code_t::OK, ""}; + } + + virtual error_t + collect_pass_by_value_tensors_node( + std::unordered_map &pass_by_values) const override final { + for (auto [uid, value] : deserialized_pass_by_value) { + pass_by_values.emplace(uid, value); + } + return {error_code_t::OK, ""}; + } + + virtual error_t + collect_tensors_in_workspace_node( + std::unordered_map>> + &worskspace_modifications, + int64_t &) const override { + for (auto [uid, value] : deserialized_workspace_modifications) { + worskspace_modifications.emplace(uid, value); + } + return {error_code_t::OK, ""}; + } + + virtual error_t + create_cudnn_tensors_node(std::unordered_map> &, + int64_t &, + std::unordered_set const &) const override final { + return {error_code_t::OK, ""}; + } + + error_t + extend_tensor_map_with_workspace_tensors_( + std::unordered_map &tensor_to_pointer_map, + void *workspace, + std::unordered_map>> const &worskspace_modifications) + const { + for (auto const &[uid, data] : worskspace_modifications) { + tensor_to_pointer_map.emplace(uid, static_cast(workspace) + std::get<1>(data)); + } + return {error_code_t::OK, ""}; + } + + error_t + extend_tensor_map_with_pass_by_value_tensors_( + std::unordered_map &tensor_to_pointer_map, + std::unordered_map &tensor_to_pass_by_value) const { + for (auto &[uid, value] : tensor_to_pass_by_value) { + if (half *half_value_ptr = std::get_if(&value)) { + tensor_to_pointer_map.emplace(uid, half_value_ptr); + } else if (nv_bfloat16 *nv_bfloat16_value_ptr = std::get_if(&value)) { + tensor_to_pointer_map.emplace(uid, nv_bfloat16_value_ptr); + } else if (int32_t *int32_t_value_ptr = std::get_if(&value)) { + tensor_to_pointer_map.emplace(uid, int32_t_value_ptr); + } else if (int64_t *int64_t_value_ptr = std::get_if(&value)) { + tensor_to_pointer_map.emplace(uid, int64_t_value_ptr); + } else if (float *float_value_ptr = std::get_if(&value)) { + tensor_to_pointer_map.emplace(uid, float_value_ptr); + } else if (double *double_value_ptr = std::get_if(&value)) { + tensor_to_pointer_map.emplace(uid, double_value_ptr); + } else { + RETURN_CUDNN_FRONTEND_ERROR_IF( + true, error_code_t::INVALID_VARIANT_PACK, "Unexpected type for pass by value tensor."); + } + } + return {error_code_t::OK, ""}; + } + + error_t + make_variant_pack_replacements( + std::unordered_map &tensor_to_pointer_map, + std::unordered_map> replacements) const { + for (auto &[from_uid, value] : replacements) { + const auto &[to_uid, start_offset] = value; + + // Check if from_uid exists in the map + auto it = tensor_to_pointer_map.find(from_uid); + RETURN_CUDNN_FRONTEND_ERROR_IF(it == tensor_to_pointer_map.end(), + error_code_t::INVALID_VARIANT_PACK, + "Variant pack expected uid " + std::to_string(from_uid) + " but not found."); + + // Perform pointer arithmetic + tensor_to_pointer_map[to_uid] = static_cast(static_cast(it->second) + start_offset); + } + return {error_code_t::OK, ""}; + } + + int64_t + get_max_cudnn_workspace_size() const { + return get_max_cudnn_workspace_size_node(); + } + + // Key: uid to replace in variant pack + // Value: uid to replace with, start offset to add to pointer + std::unordered_map> + variant_pack_replacements; + + error_t + run_auxiliary_kernels( + cudnnHandle_t handle, + void *fe_workspace, + std::unordered_map>> &workspace_modifications) const { + cudaStream_t stream; + _CUDNN_CHECK_CUDNN_ERROR(detail::get_stream(handle, &stream)); + char *workspace = static_cast(fe_workspace); + + for (auto [uid, data] : workspace_modifications) { + (void)uid; + if (std::get<0>(data) == 0) { + auto &vec_data = std::get<2>(data); + _CUDNN_CHECK_CUDA_ERROR(detail::cuda_mem_cpy_async(workspace + std::get<1>(data), + vec_data.data(), + vec_data.size() * sizeof(float), + cudaMemcpyHostToDevice, + stream)); + } else if (std::get<0>(data) == 1) { + int64_t memset_size = (int64_t)std::get<2>(data)[0]; + _CUDNN_CHECK_CUDA_ERROR( + detail::cuda_mem_set_async(workspace + std::get<1>(data), 0, memset_size, stream)); + } + } + return {error_code_t::OK, ""}; + } + + size_t + key(bool remove_shape) { +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + json j; + serialize(j); + j.erase("graph_uid"); + if (remove_shape) { + for (auto &tensor : j["tensors"]) { + tensor["dim"].clear(); + tensor["stride"].clear(); + } + } + return std::hash{}(j); +#else + CUDNN_FRONTEND_UNUSED(remove_shape); + return 1; +#endif + } + + // Private unified sdpa method - internal implementation for both FP16 and FP8 modes + inline SDPA_attributes::SDPA_outputs + sdpa_internal(std::shared_ptr q, + std::shared_ptr k, + std::shared_ptr v, + SDPA_attributes &&attributes) { + // Set inputs + attributes.inputs[SDPA_attributes::input_names::Q] = q; + attributes.inputs[SDPA_attributes::input_names::K] = k; + attributes.inputs[SDPA_attributes::input_names::V] = v; + + // Make required output tensors + SDPA_attributes::SDPA_outputs sdpa_outputs; + + sdpa_outputs.O = attributes.outputs[SDPA_attributes::output_names::O] = output_tensor(attributes.name + "::O"); + + if (attributes.generate_stats == true) { + sdpa_outputs.Stats = attributes.outputs[SDPA_attributes::output_names::Stats] = + output_tensor(attributes.name + "::Stats"); + } + + // Dropout mask dump (created conditionally based on dropout parameters) + if (attributes.outputs.find(SDPA_attributes::output_names::RNG_DUMP) != attributes.outputs.end() && + attributes.outputs.at(SDPA_attributes::output_names::RNG_DUMP) != nullptr) { + sdpa_outputs.RNG_DUMP = attributes.outputs[SDPA_attributes::output_names::RNG_DUMP]; + } + + // FP8-specific outputs (created conditionally based on FP8 scaling parameters) + if (attributes.inputs.find(SDPA_attributes::input_names::Descale_S) != attributes.inputs.end() && + attributes.inputs.at(SDPA_attributes::input_names::Descale_S) != nullptr) { + sdpa_outputs.Amax_S = attributes.outputs[SDPA_attributes::output_names::Amax_S] = + output_tensor(attributes.name + "::Amax_S"); + } + if (attributes.mma_core_mode == DataType_t::FP8_E4M3 || attributes.mma_core_mode == DataType_t::FP8_E5M2) { + sdpa_outputs.Amax_O = attributes.outputs[SDPA_attributes::output_names::Amax_O] = + output_tensor(attributes.name + "::Amax_O"); + } + + if (attributes.implementation == AttentionImplementation_t::AUTO) { + // Sets attributes.implementation to a supporting implementation, + // or leaves as AUTO if none found + attributes._auto_select_implementation(context); + } + + switch (attributes.implementation) { + case AttentionImplementation_t::AUTO: + throw std::runtime_error("No suitable implementation for given SDPA_attributes"); + break; + case AttentionImplementation_t::COMPOSITE: + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + break; + case AttentionImplementation_t::UNIFIED: + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + break; + } + + return sdpa_outputs; + } + + // Register an OSS NVRTC engine for SDPA by extracting tensor metadata from the SDPA node's attributes + error_t + register_oss_engine_() { + // Find the SDPA node in the graph's sub_nodes via dynamic_cast + SDPA_attributes const *sdpa_attrs = nullptr; + for (auto const &sub_node : sub_nodes) { + if (auto *composite = dynamic_cast(sub_node.get())) { + sdpa_attrs = &composite->attributes; + break; + } + if (auto *unified = dynamic_cast(sub_node.get())) { + sdpa_attrs = &unified->attributes; + break; + } + } + + RETURN_CUDNN_FRONTEND_ERROR_IF( + sdpa_attrs == nullptr, error_code_t::GRAPH_NOT_SUPPORTED, "No SDPA node found for OPENSOURCE engine"); + + graph::Execution_plan_list::OssSdpaEngineContext ctx; + + // Q tensor + auto q_it = sdpa_attrs->inputs.find(SDPA_attributes::input_names::Q); + RETURN_CUDNN_FRONTEND_ERROR_IF(q_it == sdpa_attrs->inputs.end() || !q_it->second, + error_code_t::GRAPH_NOT_SUPPORTED, + "Q tensor not found in SDPA node"); + auto const &q = q_it->second; + ctx.q_uid = q->get_uid(); + ctx.q_stride = q->get_stride(); + ctx.batch = q->get_dim()[0]; + ctx.heads_q = q->get_dim()[1]; + ctx.seq_q = q->get_dim()[2]; + ctx.d = q->get_dim()[3]; + + // K tensor — CompositeSDPANode transposes K in-place (swaps dims[2]/dims[3] + // and strides[2]/strides[3]) for the Q*K^T GEMM. Detect and undo. + auto k_it = sdpa_attrs->inputs.find(SDPA_attributes::input_names::K); + RETURN_CUDNN_FRONTEND_ERROR_IF(k_it == sdpa_attrs->inputs.end() || !k_it->second, + error_code_t::GRAPH_NOT_SUPPORTED, + "K tensor not found in SDPA node"); + auto const &k = k_it->second; + auto const &k_dims = k->get_dim(); + auto const &k_strides = k->get_stride(); + ctx.k_uid = k->get_uid(); + ctx.heads_kv = k_dims[1]; + if (k_dims[2] < k_dims[3]) { + // dims currently (B, H_kv, D, S_kv) — swapped + ctx.seq_kv = k_dims[3]; + ctx.k_stride = {k_strides[0], k_strides[1], k_strides[3], k_strides[2]}; + } else { + ctx.seq_kv = k_dims[2]; + ctx.k_stride = k_strides; + } + + // V tensor + auto v_it = sdpa_attrs->inputs.find(SDPA_attributes::input_names::V); + RETURN_CUDNN_FRONTEND_ERROR_IF(v_it == sdpa_attrs->inputs.end() || !v_it->second, + error_code_t::GRAPH_NOT_SUPPORTED, + "V tensor not found in SDPA node"); + ctx.v_uid = v_it->second->get_uid(); + ctx.v_stride = v_it->second->get_stride(); + + // O tensor + auto o_it = sdpa_attrs->outputs.find(SDPA_attributes::output_names::O); + RETURN_CUDNN_FRONTEND_ERROR_IF(o_it == sdpa_attrs->outputs.end() || !o_it->second, + error_code_t::GRAPH_NOT_SUPPORTED, + "O tensor not found in SDPA node"); + ctx.o_uid = o_it->second->get_uid(); + ctx.o_stride = o_it->second->get_stride(); + + // Max tensor (optional — only present if user called set_logit_max) + auto max_it = sdpa_attrs->outputs.find(SDPA_attributes::output_names::Max); + if (max_it != sdpa_attrs->outputs.end() && max_it->second) { + ctx.max_uid = max_it->second->get_uid(); + ctx.max_stride = max_it->second->get_stride(); + } + + // Sum_exp tensor (optional — only present if user called set_score_sum_exp) + auto se_it = sdpa_attrs->outputs.find(SDPA_attributes::output_names::Sum_exp); + if (se_it != sdpa_attrs->outputs.end() && se_it->second) { + ctx.sum_exp_uid = se_it->second->get_uid(); + ctx.sum_exp_stride = se_it->second->get_stride(); + } + + // Attention scale (use user-provided value if set, otherwise engine computes 1/sqrt(d)) + if (sdpa_attrs->attn_scale_value.has_value()) { + ctx.attn_scale = sdpa_attrs->attn_scale_value.value(); + } + + RETURN_CUDNN_FRONTEND_ERROR_IF(ctx.q_uid == -1 || ctx.k_uid == -1 || ctx.v_uid == -1 || ctx.o_uid == -1, + error_code_t::GRAPH_NOT_SUPPORTED, + "Could not find Q/K/V/O tensors in SDPA node for OPENSOURCE engine"); + + // Detect SM version and instantiate the appropriate OSS engine + int oss_device_ordinal = 0; + experimental::detail::cuda_get_device(&oss_device_ordinal); + cudaDeviceProp oss_dev_prop; + experimental::detail::cuda_get_device_properties(&oss_dev_prop, oss_device_ordinal); + int oss_sm = oss_dev_prop.major * 10 + oss_dev_prop.minor; + + std::shared_ptr engine; + if (oss_sm / 10 == 10) { + engine = std::make_shared(); + } else { + engine = std::make_shared(); + } + plans.set_oss_sdpa_engine(engine); + plans.set_oss_sdpa_engine_context(std::move(ctx)); + + return {error_code_t::OK, ""}; + } + + // Register an OSS NVRTC engine for RmsNorm+SiLU by detecting the fusion pattern: + // RMSNormNode(X, SCALE) → Y → PointwiseNode(SWISH_FWD) → Z + // Extracts tensor metadata and instantiates the appropriate arch-specific engine. + error_t + register_oss_rms_norm_silu_engine_() { + // Scan sub_nodes for the RMSNorm → SiLU pattern + Rmsnorm_attributes const *rmsnorm_attrs = nullptr; + Pointwise_attributes const *swish_attrs = nullptr; + std::shared_ptr swish_output; + + for (size_t i = 0; i + 1 < sub_nodes.size(); ++i) { + auto *rmsnorm_node = dynamic_cast(sub_nodes[i].get()); + if (!rmsnorm_node) continue; + + auto *pointwise_node = dynamic_cast(sub_nodes[i + 1].get()); + if (!pointwise_node) continue; + + if (pointwise_node->attributes.get_mode() != PointwiseMode_t::SWISH_FWD) continue; + + // OSS engine only supports SiLU (Swish with beta=1.0). + // Reject if beta is explicitly set to a non-1.0 value. + auto beta = pointwise_node->attributes.get_swish_beta().value_or(1.0f); + if (beta != 1.0f) continue; + + // Verify the RMSNorm output feeds into the pointwise input + auto rmsnorm_y_it = rmsnorm_node->attributes.outputs.find(Rmsnorm_attributes::output_names::Y); + if (rmsnorm_y_it == rmsnorm_node->attributes.outputs.end() || !rmsnorm_y_it->second) continue; + + auto swish_in0_it = pointwise_node->attributes.inputs.find(Pointwise_attributes::input_names::IN_0); + if (swish_in0_it == pointwise_node->attributes.inputs.end() || !swish_in0_it->second) continue; + + // Check that RMSNorm output Y == SiLU input IN_0 (same tensor) + if (rmsnorm_y_it->second->get_uid() != swish_in0_it->second->get_uid()) continue; + + // Pattern matched! + rmsnorm_attrs = &rmsnorm_node->attributes; + swish_attrs = &pointwise_node->attributes; + swish_output = pointwise_node->attributes.outputs.at(Pointwise_attributes::output_names::OUT_0); + break; + } + + RETURN_CUDNN_FRONTEND_ERROR_IF(rmsnorm_attrs == nullptr || swish_attrs == nullptr, + error_code_t::GRAPH_NOT_SUPPORTED, + "No RMSNorm → SiLU fusion pattern found for OPENSOURCE engine"); + + // Only inference is supported (no mean/inv_variance output for training) + RETURN_CUDNN_FRONTEND_ERROR_IF(rmsnorm_attrs->forward_phase != NormFwdPhase_t::INFERENCE, + error_code_t::GRAPH_NOT_SUPPORTED, + "OSS RmsNorm+SiLU engine only supports INFERENCE phase"); + + // Extract tensor metadata + graph::Execution_plan_list::OssRmsNormSiluContext ctx; + + // X input tensor + auto x_it = rmsnorm_attrs->inputs.find(Rmsnorm_attributes::input_names::X); + RETURN_CUDNN_FRONTEND_ERROR_IF(x_it == rmsnorm_attrs->inputs.end() || !x_it->second, + error_code_t::GRAPH_NOT_SUPPORTED, + "X tensor not found in RMSNorm node"); + + // Input must be bf16 (the kernel always uses ITYPE = nv_bfloat16) + auto x_dtype = x_it->second->get_data_type(); + RETURN_CUDNN_FRONTEND_ERROR_IF(x_dtype != DataType_t::BFLOAT16, + error_code_t::GRAPH_NOT_SUPPORTED, + "OSS RmsNorm+SiLU engine requires BFLOAT16 input"); + + ctx.x_uid = x_it->second->get_uid(); + auto x_dim = x_it->second->get_dim(); + // X shape in NCHW: [N, C, H, W] where norm operates per-row (N) across columns (C*H*W) + // num_tokens = N (first dim), C = product of remaining dims (C*H*W) + RETURN_CUDNN_FRONTEND_ERROR_IF( + x_dim.size() < 2, error_code_t::GRAPH_NOT_SUPPORTED, "X tensor must have at least 2 dimensions"); + ctx.num_tokens = x_dim[0]; + ctx.C = 1; + for (size_t d = 1; d < x_dim.size(); ++d) { + ctx.C *= x_dim[d]; + } + RETURN_CUDNN_FRONTEND_ERROR_IF( + ctx.num_tokens <= 0 || ctx.C <= 0, + error_code_t::GRAPH_NOT_SUPPORTED, + "Invalid problem dimensions: num_tokens=" + std::to_string(ctx.num_tokens) + " C=" + std::to_string(ctx.C)); + + // SCALE (gamma) tensor — must be bf16 (kernel uses WTYPE = nv_bfloat16) + auto scale_it = rmsnorm_attrs->inputs.find(Rmsnorm_attributes::input_names::SCALE); + RETURN_CUDNN_FRONTEND_ERROR_IF(scale_it == rmsnorm_attrs->inputs.end() || !scale_it->second, + error_code_t::GRAPH_NOT_SUPPORTED, + "SCALE tensor not found in RMSNorm node"); + RETURN_CUDNN_FRONTEND_ERROR_IF(scale_it->second->get_data_type() != DataType_t::BFLOAT16, + error_code_t::GRAPH_NOT_SUPPORTED, + "OSS RmsNorm+SiLU engine requires BFLOAT16 scale weights"); + ctx.scale_uid = scale_it->second->get_uid(); + + // BIAS (beta) tensor — optional + auto bias_it = rmsnorm_attrs->inputs.find(Rmsnorm_attributes::input_names::BIAS); + if (bias_it != rmsnorm_attrs->inputs.end() && bias_it->second) { + ctx.bias_uid = bias_it->second->get_uid(); + } + + // EPSILON tensor + auto eps_it = rmsnorm_attrs->inputs.find(Rmsnorm_attributes::input_names::EPSILON); + RETURN_CUDNN_FRONTEND_ERROR_IF(eps_it == rmsnorm_attrs->inputs.end() || !eps_it->second, + error_code_t::GRAPH_NOT_SUPPORTED, + "EPSILON tensor not found in RMSNorm node"); + ctx.epsilon_uid = eps_it->second->get_uid(); + + // Output Z tensor (output of SiLU, not RMSNorm) + RETURN_CUDNN_FRONTEND_ERROR_IF( + !swish_output, error_code_t::GRAPH_NOT_SUPPORTED, "Output tensor not found in SiLU node"); + ctx.y_uid = swish_output->get_uid(); + + // Determine output dtype from the output tensor + auto out_dtype = swish_output->get_data_type(); + if (out_dtype == DataType_t::BFLOAT16) { + ctx.output_dtype = experimental::RmsNormSiluDtype::BF16; + } else if (out_dtype == DataType_t::FP8_E4M3) { + ctx.output_dtype = experimental::RmsNormSiluDtype::FP8; + } else if (out_dtype == DataType_t::FP4_E2M1) { + ctx.output_dtype = experimental::RmsNormSiluDtype::NVFP4; + } else { + return {error_code_t::GRAPH_NOT_SUPPORTED, + "OSS RmsNorm+SiLU engine supports BFLOAT16, FP8_E4M3, or FP4_E2M1 output"}; + } + + // Detect SM version and instantiate the appropriate OSS engine + int oss_device_ordinal = 0; + experimental::detail::cuda_get_device(&oss_device_ordinal); + cudaDeviceProp oss_dev_prop; + experimental::detail::cuda_get_device_properties(&oss_dev_prop, oss_device_ordinal); + int oss_sm = oss_dev_prop.major * 10 + oss_dev_prop.minor; + + std::shared_ptr engine; + if (oss_sm >= 80) { + engine = std::make_shared(); + } else { + return {error_code_t::GRAPH_NOT_SUPPORTED, + "RmsNorm+SiLU OSS engine requires SM80+, got SM" + std::to_string(oss_sm)}; + } + + plans.set_oss_rms_norm_silu_engine(engine); + plans.set_oss_rms_norm_silu_context(std::move(ctx)); + + return {error_code_t::OK, ""}; + } + + public: + Graph() : INode(detail::Context{}) { + static std::atomic next_graph_uid{1}; + graph_uid = next_graph_uid.fetch_add(1, std::memory_order_relaxed); + } + + error_t + update_cuda_graph(cudnnHandle_t handle, + std::unordered_map, void *> &tensor_to_pointer_map, + void *workspace, + cudaGraph_t cudnn_cuda_graph) { + // First get all the uids from the map + std::unordered_map tensor_uid_to_pointer_map; + tensor_uid_to_pointer_map.reserve(tensor_to_pointer_map.size()); + for (auto const &[tensor, pointer] : tensor_to_pointer_map) { + tensor_uid_to_pointer_map.emplace(tensor->get_uid(), pointer); + } + + return update_cuda_graph(handle, tensor_uid_to_pointer_map, workspace, cudnn_cuda_graph); + } + + error_t + update_cuda_graph(cudnnHandle_t handle, + std::unordered_map &uid_to_device_ptrs, + void *workspace, + cudaGraph_t cudnn_cuda_graph) { + // Initializes this cudnn graph + RETURN_CUDNN_FRONTEND_ERROR_IF( + cudnn_cuda_graph == nullptr, error_code_t::INVALID_VALUE, "cudnn_cuda_graph should not be a nullptr"); + + size_t num_root_nodes; + _CUDNN_CHECK_CUDA_ERROR(detail::cuda_graph_get_root_nodes(cudnn_cuda_graph, nullptr, &num_root_nodes)); + RETURN_CUDNN_FRONTEND_ERROR_IF( + num_root_nodes != 1, error_code_t::INVALID_VALUE, "cudnn_cuda_graph should have exactly 1 root node."); + + cudaGraphNode_t current_node = nullptr; + _CUDNN_CHECK_CUDA_ERROR(detail::cuda_graph_get_root_nodes(cudnn_cuda_graph, ¤t_node, &num_root_nodes)); + + /////////////////////////////////////// + //// PASS BY VALUE TENSOR HANDLING //// + /////////////////////////////////////// + // Add pass_by_value data pointers to uid_to_pointer map. + // Using cached values to avoid repeated tree traversal overhead. + // cuda graph will keep a copy of the kernel parameters, meaning that at the time of + // launching the cuda_graph executable, cached values being deallocated does not affect these cpu values. + // No cuda graph nodes are required for handling fe owned pass by value tensors. + CHECK_CUDNN_FRONTEND_ERROR( + extend_tensor_map_with_pass_by_value_tensors_(uid_to_device_ptrs, cached_pass_by_value)); + + //////////////////////////// + //// WORKSPACE HANDLING //// + //////////////////////////// + // Using cached workspace modifications to avoid repeated tree traversal. + for (auto const &[uid, data] : cached_workspace_modifications) { + const auto &[operation_type, offset, vec_data] = data; + uid_to_device_ptrs[uid] = static_cast(workspace) + offset; + + // 0 means memcpy + if (operation_type == 0) { + _CUDNN_CHECK_CUDA_ERROR( + detail::cuda_graph_add_memcpy_node_set_params_1D(current_node, + static_cast(workspace) + offset, + vec_data.data(), + vec_data.size() * sizeof(float), + cudaMemcpyHostToDevice)); + } + // 1 means memset + else if (operation_type == 1) { + // offset from workspace + void *device_ptr = static_cast(workspace) + offset; + int64_t memset_size = static_cast(vec_data[0]); + + cudaMemsetParams params; + params.dst = device_ptr; + params.elementSize = sizeof(char); + params.value = 0x0; + params.width = memset_size; + params.height = 1; // 1D memset currently + params.pitch = 0; // unused + + _CUDNN_CHECK_CUDA_ERROR(detail::cuda_graph_add_memset_node_set_params(current_node, ¶ms)); + } + // Other values do not correspond to CUDA graph nodes + else { + continue; + } + + size_t num_dependent_nodes; + _CUDNN_CHECK_CUDA_ERROR( + detail::cuda_graph_node_get_dependent_nodes(current_node, nullptr, &num_dependent_nodes)); + RETURN_CUDNN_FRONTEND_ERROR_IF( + num_dependent_nodes != 1, + error_code_t::INVALID_VALUE, + "Each node of cudnn_cuda_graph before the backend graph node should have exactly 1 dependent node."); + _CUDNN_CHECK_CUDA_ERROR( + detail::cuda_graph_node_get_dependent_nodes(current_node, ¤t_node, &num_dependent_nodes)); + } + + // Make sure device pointer is provided for all uids expected for this plan + std::vector device_ptrs; + std::vector uids; + + device_ptrs.reserve(variant_pack_uids.size()); + uids.reserve(variant_pack_uids.size()); + + for (auto const &uid : variant_pack_uids) { + auto search = uid_to_device_ptrs.find(uid); + RETURN_CUDNN_FRONTEND_ERROR_IF(search == uid_to_device_ptrs.end(), + error_code_t::INVALID_VARIANT_PACK, + "Uid " + std::to_string(uid) + " does not exist in variant pack."); + device_ptrs.push_back(search->second); + uids.push_back(uid); + } + + /////////////////// + //// BE GRAPH //// + /////////////////// + cudaGraph_t backend_cuda_graph; + _CUDNN_CHECK_CUDA_ERROR(detail::cuda_graph_child_graph_node_get_graph(current_node, &backend_cuda_graph)); + + detail::backend_descriptor variant_pack_descriptor(CUDNN_BACKEND_VARIANT_PACK_DESCRIPTOR); + RETURN_CUDNN_FRONTEND_ERROR_IF(variant_pack_descriptor.get_status() != CUDNN_STATUS_SUCCESS, + error_code_t::CUDNN_BACKEND_API_FAILED, + "Failed to create variant pack's backend descriptor."); + + // offset workspace by the already used fe graph workspace + // this is where cudnn backend can start using workspace for its execution plans + void *cudnn_workspace = static_cast(workspace) + fe_workspace_size; + CHECK_CUDNN_FRONTEND_ERROR(create_variant_pack(variant_pack_descriptor, device_ptrs, uids, cudnn_workspace)); + + int64_t candidate = plans.candidate; + CHECK_CUDNN_FRONTEND_ERROR(plans.is_plan_index_executable(candidate)); + _CUDNN_CHECK_CUDNN_ERROR(detail::update_cuda_graph(handle, + plans.execution_plans[candidate]->get_raw_desc(), + variant_pack_descriptor.get_ptr(), + backend_cuda_graph)); + + // There should be nothing after the backend graph + size_t num_dependent_nodes; + _CUDNN_CHECK_CUDA_ERROR( + detail::cuda_graph_node_get_dependent_nodes(current_node, nullptr, &num_dependent_nodes)); + RETURN_CUDNN_FRONTEND_ERROR_IF(num_dependent_nodes != 0, + error_code_t::INVALID_VALUE, + "cudnn_cuda_graph should have no graph nodes after the backend graph node."); + + return {error_code_t::OK, ""}; + } + + error_t + populate_cuda_graph(cudnnHandle_t handle, + std::unordered_map, void *> &tensor_to_pointer_map, + void *workspace, + cudaGraph_t cudnn_cuda_graph) { + // First get all the uids from the map + std::unordered_map tensor_uid_to_pointer_map; + tensor_uid_to_pointer_map.reserve(tensor_to_pointer_map.size()); + for (auto const &[tensor, pointer] : tensor_to_pointer_map) { + tensor_uid_to_pointer_map.emplace(tensor->get_uid(), pointer); + } + + return populate_cuda_graph(handle, tensor_uid_to_pointer_map, workspace, cudnn_cuda_graph); + } + + error_t + populate_cuda_graph(cudnnHandle_t handle, + std::unordered_map &uid_to_device_ptrs, + void *workspace, + cudaGraph_t cudnn_cuda_graph) { + // Check if the cuda graph is empty + size_t numNodes = 0; + _CUDNN_CHECK_CUDA_ERROR(detail::cuda_graph_get_nodes(cudnn_cuda_graph, nullptr, &numNodes)); + RETURN_CUDNN_FRONTEND_ERROR_IF(numNodes != 0, + error_code_t::INVALID_VALUE, + "cuda graph provided to populate is not empty. cuDNN requires it to be empty " + "for the corresponding update APIs to work correctly."); + + // This function makes linear cuda graphs. And that makes it easy to walk + // the graph when updating it. + // So just keeping track of the last node in the cuda graph is sufficient. + cudaGraphNode_t last_node = nullptr; + + /////////////////////////////////////// + //// PASS BY VALUE TENSOR HANDLING //// + /////////////////////////////////////// + // Add pass_by_value data pointers to uid_to_pointer map. + // Using cached values to avoid repeated tree traversal overhead. + // cuda graph will keep a copy of the kernel parameters, meaning that at the time of + // launching the cuda_graph executable, cached values being deallocated does not affect these cpu values. + // No cuda graph nodes are required for handling fe owned pass by value tensors. + CHECK_CUDNN_FRONTEND_ERROR( + extend_tensor_map_with_pass_by_value_tensors_(uid_to_device_ptrs, cached_pass_by_value)); + + ///////////////////////////////// + //// WORKSPACE HANDLING //// + ///////////////////////////////// + // Using cached workspace modifications to avoid repeated tree traversal. + for (auto const &[uid, data] : cached_workspace_modifications) { + const auto &[operation_type, offset, vec_data] = data; + uid_to_device_ptrs[uid] = static_cast(workspace) + offset; + + cudaGraphNode_t node = nullptr; + + // 0 means memcpy + if (operation_type == 0) { + _CUDNN_CHECK_CUDA_ERROR(detail::cuda_graph_add_memcpy_node_1D(&node, + cudnn_cuda_graph, + &last_node, + last_node != nullptr, + static_cast(workspace) + offset, + vec_data.data(), + vec_data.size() * sizeof(float), + cudaMemcpyHostToDevice)); + } + // 1 means memset + else if (operation_type == 1) { + // offset from workspace + void *device_ptr = static_cast(workspace) + offset; + int64_t memset_size = static_cast(vec_data[0]); + + cudaMemsetParams params; + params.dst = device_ptr; + params.elementSize = sizeof(char); + params.value = 0x0; + params.width = memset_size; + params.height = 1; // 1D memset currently + params.pitch = 0; // unused + + _CUDNN_CHECK_CUDA_ERROR(detail::cuda_graph_add_memset_node( + &node, cudnn_cuda_graph, &last_node, last_node != nullptr, ¶ms)); + } + // Other values do not correspond to CUDA graph nodes + else { + continue; + } + + last_node = node; + } + + ////////////// + // BE graph // + ////////////// + + // Get the BE's cuda graph + + // Make sure device pointer is provided for all uids expected for this plan + std::vector device_ptrs; + device_ptrs.reserve(variant_pack_uids.size()); + std::vector uids; + uids.reserve(variant_pack_uids.size()); + for (auto const &uid : variant_pack_uids) { + auto search = uid_to_device_ptrs.find(uid); + RETURN_CUDNN_FRONTEND_ERROR_IF(search == uid_to_device_ptrs.end(), + error_code_t::INVALID_VARIANT_PACK, + "Uid " + std::to_string(uid) + " does not exist in variant pack."); + device_ptrs.push_back(search->second); + uids.push_back(uid); + } + + // Create the variant pack to pass to backend + detail::backend_descriptor variant_pack_descriptor(CUDNN_BACKEND_VARIANT_PACK_DESCRIPTOR); + RETURN_CUDNN_FRONTEND_ERROR_IF(variant_pack_descriptor.get_status() != CUDNN_STATUS_SUCCESS, + error_code_t::CUDNN_BACKEND_API_FAILED, + "Failed to create variant pack's backend descriptor."); + + // offset workspace by the already used fe graph workspace + // this is where cudnn backend can start using workspace for its execution plans + void *cudnn_workspace = static_cast(workspace) + fe_workspace_size; + CHECK_CUDNN_FRONTEND_ERROR(create_variant_pack(variant_pack_descriptor, device_ptrs, uids, cudnn_workspace)); + + // Get the plan candidate. It only makes to sense to make cuda graph after execution plan has been built. + // And in that case the candidate would have been set. + int64_t candidate = plans.candidate; + CHECK_CUDNN_FRONTEND_ERROR(plans.is_plan_index_executable(candidate)); + + // Finally get the backend cuda graph. + cudaGraph_t backend_cuda_graph; + // Initialize the cudnn cuda graph. + // The responsibility to destroy is on the user. + _CUDNN_CHECK_CUDA_ERROR(detail::cuda_graph_create(&backend_cuda_graph, 0)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::populate_cuda_graph(handle, + plans.execution_plans[candidate]->get_raw_desc(), + variant_pack_descriptor.get_ptr(), + backend_cuda_graph)); + + // Clone BE graph into a graph_node + // This same call also places the newly created into FE's graph + // TODO: BE graph is at the end, so put in appropriate dependencies + cudaGraphNode_t backend_cuda_graph_node; + detail::cuda_graph_add_child_graph_node( + &backend_cuda_graph_node, cudnn_cuda_graph, &last_node, last_node != nullptr, backend_cuda_graph); + + // Destroy the BE graph as it now has been cloned into a node + // It was initialized by internals of backend, but the responsibility to destroy it is on FE. + _CUDNN_CHECK_CUDA_ERROR(detail::cuda_graph_destroy(backend_cuda_graph)); + + return {error_code_t::OK, ""}; + } + + error_t + validate() { + CUDNN_FE_LOG_BANNER(" VALIDATING GRAPH "); + CUDNN_FE_LOG(*this << std::endl;); + + // First validate all inputs that the user set. + for (auto const &input : full_graph_inputs) { + CHECK_CUDNN_FRONTEND_ERROR(input->validate()); + } + + // Validate the nodes, which in turn also infers missing tensor attributes. + CHECK_CUDNN_FRONTEND_ERROR(validate_subtree()); + // Validate all outputs, which should now have everything set to be lowered to backend. + for (auto const &output : full_graph_outputs) { + CHECK_CUDNN_FRONTEND_ERROR(output->validate()); + } + + // Get all the pre assigned uids + CHECK_CUDNN_FRONTEND_ERROR(get_pre_assigned_uids(used_uids)); + // Clear state + used_uids.clear(); + + CUDNN_FE_LOG_BANNER(" VALIDATED ALL OK "); + + return {error_code_t::OK, ""}; + } + + // overload for deviceless AoT compilation + error_t + build_operation_graph() { + CUDNN_FE_LOG_BANNER(" BUILD OP GRAPH WITHOUT HANDLE "); + + if (device_properties == nullptr) { + return {error_code_t::ATTRIBUTE_NOT_SET, "Device properties are not set."}; + } + CUDNN_FE_LOG_BANNER(" BUILT OP GRAPH WITHOUT HANDLE "); + return build_operation_graph(nullptr); + } + + error_t + build_operation_graph(cudnnHandle_t handle) { + CUDNN_FE_LOG_BANNER(" BUILD OP GRAPH "); + + CUDNN_FE_LOG_BANNER(" 1/4 INFER PROPERTIES OF NODES "); + + // expand composite nodes + CHECK_CUDNN_FRONTEND_ERROR(expand_subtree()); + + // Get all the pre assigned uids + CHECK_CUDNN_FRONTEND_ERROR(get_pre_assigned_uids(used_uids)); + + CUDNN_FE_LOG_BANNER(" 2/4 CREATE TENSORS "); + + Tensor_attributes::uid_t start_uid = 1; + CHECK_CUDNN_FRONTEND_ERROR(create_cudnn_tensors_subtree(uid_to_tensors, start_uid, used_uids)); + tensors_to_dump.clear(); + CHECK_CUDNN_FRONTEND_ERROR(collect_tensors_to_dump_subtree(tensors_to_dump)); + + CUDNN_FE_LOG_BANNER(" 3/4 CREATE OPERATIONS "); + // INode keeps track of all uids that an operation graph uses. + // This helps to return errors to user during execution, without relying on backend to do so. + // Also, as uid in a variant pack have to be unique, keep a set of them. + CHECK_CUDNN_FRONTEND_ERROR( + create_cudnn_operations(variant_pack_uids, operations, raw_operations, uid_to_tensors)); + + // Collect variant pack modifiers when lowering to backend. + // The collected map is used everytime when execute is called. + CHECK_CUDNN_FRONTEND_ERROR(collect_variant_pack_replacements_subtree(variant_pack_replacements)); + + fe_workspace_size = get_fe_workspace_size_subtree(); + + // Cache pass_by_value tensors and workspace modifications for fast execution. + // These are collected once here and reused in every execute() call to avoid + // repeated tree traversal and map allocation overhead. + CHECK_CUDNN_FRONTEND_ERROR(collect_pass_by_value_tensors_subtree(cached_pass_by_value)); + { + int64_t temp_offset = 0; + CHECK_CUDNN_FRONTEND_ERROR( + collect_tensors_in_workspace_subtree(cached_workspace_modifications, temp_offset)); + } + + CUDNN_FE_LOG_BANNER(" 4/4 LOWERING TO BACKEND OPERATION GRAPH "); + + // The method here fuses all operations. There will be 1 operation graph in total. + CHECK_CUDNN_FRONTEND_ERROR(create_cudnn_operation_graph(handle)); + + if (context.get_dynamic_shape_enabled() && kernel_cache && !kernel_cache->is_finalized()) { + CUDNN_FE_LOG_BANNER(" BUILD KERNEL CACHE "); + CHECK_CUDNN_FRONTEND_ERROR(kernel_cache->build(operation_graph->get_raw_desc())); + } + + CUDNN_FE_LOG_BANNER(" BUILD OP GRAPH ALL OK === "); + + return {error_code_t::OK, ""}; + } + + error_t + get_plan_name(std::string &name) const { + return get_plan_name_at_index(plans.candidate, name); + } + + error_t + get_plan_name_at_index(int64_t plan_index, std::string &name) const { + auto ret_val = plans.get_name_at_index(plan_index, name); + CUDNN_FE_LOG_LABEL_ENDL("INFO: get_plan_name_at_index(" << plan_index << ") is " + name); + return ret_val; + } + + // Structured counterpart of get_plan_name_at_index(): engine index + knob + // choices, for replay via create_execution_plan(). + error_t + get_engine_and_knobs_at_index(int64_t plan_index, + int64_t &engine_id, + std::unordered_map &knobs) const { + return plans.get_engine_and_knobs_at_index(plan_index, engine_id, knobs); + } + + error_t + get_workspace_size(int64_t &cudnn_workspace_size) const { + return get_workspace_size_plan_at_index(plans.candidate, cudnn_workspace_size); + } + + error_t + get_workspace_size(cudnnHandle_t handle, + int64_t &cudnn_workspace_size, + std::vector const &override_uids, + std::vector> const &override_shapes, + std::vector> const &override_strides) const { + return get_workspace_size_plan_at_index( + handle, plans.candidate, cudnn_workspace_size, override_uids, override_shapes, override_strides); + } + + error_t + get_workspace_size_plan_at_index(int64_t plan_index, int64_t &cudnn_workspace_size) const { + // OSS SDPA engine workspace: 16 bytes for tile_id_counter + if (plan_index == graph::Execution_plan_list::OSS_SDPA_ENGINE_CANDIDATE) { + cudnn_workspace_size = fe_workspace_size + experimental::Sm90SdpaPrefillEngine::get_workspace_size(); + CUDNN_FE_LOG_LABEL_ENDL("INFO: get_workspace_size() is " << cudnn_workspace_size << " (OSS SDPA engine)"); + return {error_code_t::OK, ""}; + } + + // OSS RmsNorm+SiLU engine workspace + if (plan_index == graph::Execution_plan_list::OSS_RMS_NORM_SILU_ENGINE_CANDIDATE) { + cudnn_workspace_size = fe_workspace_size + plans.get_oss_rms_norm_silu_workspace_size(); + CUDNN_FE_LOG_LABEL_ENDL("INFO: get_workspace_size() is " << cudnn_workspace_size + << " (OSS RmsNorm+SiLU engine)"); + return {error_code_t::OK, ""}; + } + + // There are two workspaces: + // - cudnn execution plan workspace + // - FE node workspace (example: alibiSlope for fmha) + int64_t cudnn_ws = 0; + CHECK_CUDNN_FRONTEND_ERROR(get_cudnn_workspace_size_node(plan_index, cudnn_ws)); + cudnn_workspace_size = cudnn_ws + fe_workspace_size; + CUDNN_FE_LOG_LABEL_ENDL("INFO: get_workspace_size() is " << cudnn_workspace_size); + return {error_code_t::OK, ""}; + } + + error_t + get_workspace_size_plan_at_index(cudnnHandle_t handle, + int64_t plan_index, + int64_t &cudnn_workspace_size, + std::vector const &override_uids, + std::vector> const &override_shapes, + std::vector> const &override_strides) const { + RETURN_CUDNN_FRONTEND_ERROR_IF(override_uids.size() != override_shapes.size(), + error_code_t::INVALID_VALUE, + "override_uids and override_shapes must have the same size."); + RETURN_CUDNN_FRONTEND_ERROR_IF(override_uids.size() != override_strides.size(), + error_code_t::INVALID_VALUE, + "override_uids and override_strides must have the same size."); + + if (override_uids.empty()) { + return get_workspace_size_plan_at_index(plan_index, cudnn_workspace_size); + } + + // OSS engines are not backed by cuDNN execution plans, so their workspace stays frontend-owned. + if (plan_index == graph::Execution_plan_list::OSS_SDPA_ENGINE_CANDIDATE || + plan_index == graph::Execution_plan_list::OSS_RMS_NORM_SILU_ENGINE_CANDIDATE) { + return get_workspace_size_plan_at_index(plan_index, cudnn_workspace_size); + } + + auto cudnn_ver_error = error_t{error_code_t::GRAPH_NOT_SUPPORTED, + "Runtime workspace query with override shapes requires cuDNN v9.23.0"}; + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(92300, cudnn_ver_error); + +#if (CUDNN_VERSION < 92300) + return cudnn_ver_error; +#endif + + CHECK_CUDNN_FRONTEND_ERROR(plans.is_plan_index_executable(plan_index)); + + detail::backend_descriptor variant_pack_descriptor(CUDNN_BACKEND_VARIANT_PACK_DESCRIPTOR); + RETURN_CUDNN_FRONTEND_ERROR_IF(variant_pack_descriptor.get_status() != CUDNN_STATUS_SUCCESS, + error_code_t::CUDNN_BACKEND_API_FAILED, + "Failed to create variant pack's backend descriptor."); + +#if (CUDNN_VERSION >= 92100) + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(variant_pack_descriptor.get_ptr(), + CUDNN_ATTR_VARIANT_PACK_OVERRIDE_UNIQUE_IDS, + CUDNN_TYPE_INT64, + override_uids.size(), + override_uids.data())); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(variant_pack_descriptor.get_ptr(), + CUDNN_ATTR_VARIANT_PACK_OVERRIDE_SHAPES, + CUDNN_TYPE_VOID_PTR, + 1, + (void *)&override_shapes)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(variant_pack_descriptor.get_ptr(), + CUDNN_ATTR_VARIANT_PACK_OVERRIDE_STRIDES, + CUDNN_TYPE_VOID_PTR, + 1, + (void *)&override_strides)); +#else + CUDNN_FRONTEND_UNUSED(override_uids); + CUDNN_FRONTEND_UNUSED(override_shapes); + CUDNN_FRONTEND_UNUSED(override_strides); +#endif + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(variant_pack_descriptor.get_ptr())); + + size_t cudnn_ws = 0; + _CUDNN_CHECK_CUDNN_ERROR(detail::get_execution_plan_workspace_size( + handle, plans.execution_plans[plan_index]->get_raw_desc(), variant_pack_descriptor.get_ptr(), &cudnn_ws)); + cudnn_workspace_size = cudnn_ws + fe_workspace_size; + CUDNN_FE_LOG_LABEL_ENDL("INFO: get_workspace_size() is " << cudnn_workspace_size + << " (runtime override shape)"); + return {error_code_t::OK, ""}; + } + + int64_t + get_workspace_size() const { + return get_workspace_size_plan_at_index(plans.candidate); + } + + int64_t + get_workspace_size(cudnnHandle_t handle, + std::vector const &override_uids, + std::vector> const &override_shapes, + std::vector> const &override_strides) const { + int64_t cudnn_workspace = 0; + auto status = get_workspace_size(handle, cudnn_workspace, override_uids, override_shapes, override_strides); + if (status.is_bad()) { + CUDNN_FE_LOG_LABEL_ENDL("ERROR: Querying workspace failed."); + } + return cudnn_workspace; + } + + int64_t + get_workspace_size_plan_at_index(int64_t plan_index) const { + int64_t cudnn_workspace = 0; + auto status = get_workspace_size_plan_at_index(plan_index, cudnn_workspace); + if (status.is_bad()) { + CUDNN_FE_LOG_LABEL_ENDL("ERROR: Querying workspace failed."); + } + return cudnn_workspace; + } + + int64_t + get_workspace_size_plan_at_index(cudnnHandle_t handle, + int64_t plan_index, + std::vector const &override_uids, + std::vector> const &override_shapes, + std::vector> const &override_strides) const { + int64_t cudnn_workspace = 0; + auto status = get_workspace_size_plan_at_index( + handle, plan_index, cudnn_workspace, override_uids, override_shapes, override_strides); + if (status.is_bad()) { + CUDNN_FE_LOG_LABEL_ENDL("ERROR: Querying workspace failed."); + } + return cudnn_workspace; + } + + int64_t + get_autotune_workspace_size() const { + // There are two workspaces: + // - cudnn execution plan workspace + // - FE node workspace (example: alibiSlope for fmha) + return fe_workspace_size + get_max_cudnn_workspace_size(); + } + + error_t + autotune(cudnnHandle_t handle, + std::unordered_map &tensor_uid_to_pointer_map, + void *workspace, + void *user_impl = nullptr) { + (void)user_impl; // reserved for future use + + const int maxIterCount = 100; + const float threshold = 0.95f; + + auto plan_cmp = [](std::shared_ptr a, std::shared_ptr b) { + return a->getExecutionTime() < b->getExecutionTime(); + }; + std::multiset, decltype(plan_cmp)> timed_plans(plan_cmp); + + cudaEvent_t start, stop; + detail::cuda_event_create(&start); + detail::cuda_event_create(&stop); + detail::cuda_device_synchronize(); + + cudaStream_t stream = nullptr; + detail::get_stream(handle, &stream); + + uint64_t successful_plan_count = 0; + for (int64_t i = 0; i < static_cast(plans.execution_plans.size()); i++) { + if (plans.execution_plans[i] == nullptr) continue; + + // Warm-up run + auto warmup_status = execute_plan_at_index(handle, tensor_uid_to_pointer_map, workspace, i); + if (warmup_status.is_bad()) { + CUDNN_FE_LOG_LABEL_ENDL("WARN: Plan " << i << " failed warmup, skipping."); + continue; + } + successful_plan_count++; + detail::cuda_device_synchronize(); + + float min_time_ms = std::numeric_limits::max(); + for (int iter = 0; iter < maxIterCount; iter++) { + detail::cuda_event_record(start, stream); + auto iter_status = execute_plan_at_index(handle, tensor_uid_to_pointer_map, workspace, i); + detail::cuda_event_record(stop, stream); + detail::cuda_event_synchronize(stop); + + if (iter_status.is_bad()) { + CUDNN_FE_LOG_LABEL_ENDL("WARN: Plan " << i << " failed at iter " << iter << ", skipping time."); + continue; + } + + float time_ms = 0.0f; + detail::cuda_event_elapsed_time(&time_ms, start, stop); + float new_min = std::min(min_time_ms, time_ms); + if (time_ms / min_time_ms < threshold) { + min_time_ms = new_min; + } else { + break; + } + } + + CUDNN_FE_LOG_LABEL_ENDL("Plan " << plans.execution_plans[i]->getTag() << " took " << std::setw(10) + << min_time_ms); + plans.execution_plans[i]->setExecutionTime(min_time_ms); + timed_plans.insert(plans.execution_plans[i]); + } + + // Re-order plans by measured time, winner at index 0 + plans.execution_plans.clear(); + for (auto sorted_plan : timed_plans) { + plans.execution_plans.push_back(sorted_plan); + } + plans.candidate = 0; + + // Re-prepare OSS slot indices to match the new plan ordering + apply_oss_slot_indices_to_plans(); + + detail::cuda_event_destroy(start); + detail::cuda_event_destroy(stop); + + CUDNN_FE_LOG_LABEL_ENDL("Autotuned " << successful_plan_count << " plans."); + return {error_code_t::OK, ""}; + } + + error_t + autotune(cudnnHandle_t handle, + std::unordered_map, void *> &tensor_to_pointer_map, + void *workspace, + void *user_impl = nullptr) { + std::unordered_map uid_map; + for (auto const &[tensor, pointer] : tensor_to_pointer_map) { + uid_map.emplace(tensor->get_uid(), pointer); + } + return autotune(handle, uid_map, workspace, user_impl); + } + + // ----------------------------------------------------------------------- + // Execute overloads. + // All roads lead to execute_plan_at_index(handle, sorted_ptrs, n, ws, plan_index). + // ----------------------------------------------------------------------- + + // --- Convenience wrappers: convert key types, use default plan index --- + + error_t + execute(cudnnHandle_t handle, + std::unordered_map, void *> &tensor_to_pointer_map, + void *workspace) const { + std::unordered_map uid_map; + for (auto const &[tensor, pointer] : tensor_to_pointer_map) { + uid_map.emplace(tensor->get_uid(), pointer); + } + return execute(handle, uid_map, workspace); + } + + error_t + execute(cudnnHandle_t handle, + std::unordered_map &tensor_uid_to_pointer_map, + void *workspace) const { + return execute_plan_at_index(handle, tensor_uid_to_pointer_map, workspace, plans.candidate); + } + + error_t + execute(cudnnHandle_t handle, + std::unordered_map &tensor_uid_to_pointer_map, + void *workspace, + std::vector const &override_uids, + std::vector> const &override_shapes, + std::vector> const &override_strides) const { + return execute_plan_at_index(handle, + tensor_uid_to_pointer_map, + workspace, + plans.candidate, + override_uids, + override_shapes, + override_strides); + } + + error_t + execute(cudnnHandle_t handle, void **sorted_user_ptrs, int n_user, void *workspace) const { + return execute_plan_at_index(handle, sorted_user_ptrs, n_user, workspace, plans.candidate); + } + + // --- execute_plan_at_index: convert key types --- + + error_t + execute_plan_at_index(cudnnHandle_t handle, + std::unordered_map, void *> &tensor_to_pointer_map, + void *workspace, + int64_t plan_index) const { + std::unordered_map uid_map; + for (auto const &[tensor, pointer] : tensor_to_pointer_map) { + uid_map.emplace(tensor->get_uid(), pointer); + } + return execute_plan_at_index(handle, uid_map, workspace, plan_index); + } + + // uid map → extract sorted ptrs, delegate to the sorted_ptrs implementation + error_t + execute_plan_at_index(cudnnHandle_t handle, + std::unordered_map &tensor_uid_to_pointer_map, + void *workspace, + int64_t plan_index, + std::vector const &override_uids = {}, + std::vector> const &override_shapes = {}, + std::vector> const &override_strides = {}) const { + if (!varpack_prep_state->prepared.load(std::memory_order_acquire)) { + CHECK_CUDNN_FRONTEND_ERROR(const_cast(this)->prepare_variant_pack_template()); + } + const int n_user = (int)varpack_template.user_slots.size(); + constexpr int STACK_MAX = 32; + void *stack_ptrs[STACK_MAX]; + std::vector heap_ptrs; + void **user_ptrs = (n_user <= STACK_MAX) ? stack_ptrs : (heap_ptrs.resize(n_user), heap_ptrs.data()); + for (int i = 0; i < n_user; i++) { + int64_t uid = varpack_template.all_uids[varpack_template.user_slots[i]]; + auto it = tensor_uid_to_pointer_map.find(uid); + RETURN_CUDNN_FRONTEND_ERROR_IF(it == tensor_uid_to_pointer_map.end(), + error_code_t::INVALID_VARIANT_PACK, + "Uid " + std::to_string(uid) + " not found in variant pack."); + user_ptrs[i] = it->second; + } + return execute_plan_at_index( + handle, user_ptrs, n_user, workspace, plan_index, override_uids, override_shapes, override_strides); + } + + // --- THE implementation: sorted pointer array + plan index + optional overrides --- + // Thread-safe: copies the pre-built template to stack, patches user pointers, dispatches. + error_t + execute_plan_at_index(cudnnHandle_t handle, + void **sorted_user_ptrs, + int n_user, + void *workspace, + int64_t plan_index, + std::vector const &override_uids = {}, + std::vector> const &override_shapes = {}, + std::vector> const &override_strides = {}) const { + // Lazy init: prepare template if not done (e.g. deserialized graphs, build_plan_at_index) + if (!varpack_prep_state->prepared.load(std::memory_order_acquire)) { + CHECK_CUDNN_FRONTEND_ERROR(const_cast(this)->prepare_variant_pack_template()); + } + + CHECK_CUDNN_FRONTEND_ERROR(plans.is_plan_index_executable(plan_index)); + + // Validate n_user matches expected user slot count + RETURN_CUDNN_FRONTEND_ERROR_IF(n_user != static_cast(varpack_template.user_slots.size()), + error_code_t::INVALID_VARIANT_PACK, + "n_user (" + std::to_string(n_user) + + ") does not match expected user slot count (" + + std::to_string(varpack_template.user_slots.size()) + ")."); + RETURN_CUDNN_FRONTEND_ERROR_IF( + n_user > 0 && sorted_user_ptrs == nullptr, error_code_t::INVALID_VARIANT_PACK, "sorted_user_ptrs is null."); + + // Thread-safe: copy template to local stack, then patch. + const int N = (int)varpack_template.all_uids.size(); + constexpr int STACK_MAX = 32; + void *stack_ptrs[STACK_MAX]; + std::vector heap_ptrs; + void **ptrs; + if (N <= STACK_MAX) { + std::memcpy(stack_ptrs, varpack_template.template_ptrs.data(), N * sizeof(void *)); + ptrs = stack_ptrs; + } else { + heap_ptrs = varpack_template.template_ptrs; + ptrs = heap_ptrs.data(); + } + + // 1. Patch user pointers + for (int i = 0; i < n_user; i++) { + ptrs[varpack_template.user_slots[i]] = sorted_user_ptrs[i]; + } + + // 2. Update workspace-relative pointers (pre-computed slot indices) + for (auto const &[slot, offset] : varpack_template.workspace_slots) { + ptrs[slot] = static_cast(workspace) + offset; + } + + // 3. Re-apply replacements (currently only Slice nodes) + for (auto const &[dst_slot, src_info] : varpack_template.replacement_slots) { + ptrs[dst_slot] = static_cast(ptrs[src_info.first]) + src_info.second; + } + + // 4. Run auxiliary kernels (e.g. SDPA reduction accumulator init) + CHECK_CUDNN_FRONTEND_ERROR(run_auxiliary_kernels(handle, workspace, cached_workspace_modifications)); + + CUDNN_FE_LOG_LABEL_ENDL("INFO: Executing graph_uid " << graph_uid); + + // 5. Dispatch + void *engine_workspace = static_cast(workspace) + fe_workspace_size; + + if (plan_index == graph::Execution_plan_list::OSS_SDPA_ENGINE_CANDIDATE) { + cudaStream_t stream = nullptr; + _CUDNN_CHECK_CUDNN_ERROR(detail::get_stream(handle, &stream)); + int device_ordinal = 0; + detail::cuda_get_device(&device_ordinal); + if (override_uids.empty()) { + CHECK_CUDNN_FRONTEND_ERROR( + plans.execute_oss_sdpa_engine(ptrs, engine_workspace, device_ordinal, stream)); + } else { + CHECK_CUDNN_FRONTEND_ERROR(plans.execute_oss_sdpa_engine( + ptrs, engine_workspace, device_ordinal, stream, override_uids, override_shapes, override_strides)); + } + return {error_code_t::OK, ""}; + } + + if (plan_index == graph::Execution_plan_list::OSS_RMS_NORM_SILU_ENGINE_CANDIDATE) { + cudaStream_t stream = nullptr; + _CUDNN_CHECK_CUDNN_ERROR(detail::get_stream(handle, &stream)); + int device_ordinal = 0; + detail::cuda_get_device(&device_ordinal); + CHECK_CUDNN_FRONTEND_ERROR( + plans.execute_oss_rms_norm_silu_engine(ptrs, engine_workspace, device_ordinal, stream)); + return {error_code_t::OK, ""}; + } + + // Backend path + if (override_uids.empty()) { + CHECK_CUDNN_FRONTEND_ERROR(detail::execute( + handle, plans.execution_plans[plan_index].get(), ptrs, varpack_template.all_uids, engine_workspace)); + } else { + CHECK_CUDNN_FRONTEND_ERROR(detail::execute(handle, + plans.execution_plans[plan_index].get(), + ptrs, + varpack_template.all_uids, + engine_workspace, + override_uids, + override_shapes, + override_strides)); + } + return {error_code_t::OK, ""}; + } + + error_t + warmup(cudnnHandle_t handle) { + cudaStream_t fake_stream; + + cudaStream_t original_stream; + + _CUDNN_CHECK_CUDNN_ERROR(detail::get_stream(handle, &original_stream)); + + CUDNN_FE_LOG_BANNER("WARMUP (BEGIN FAKE GRAPH CAPTURE) "); + + if (original_stream == nullptr) { + _CUDNN_CHECK_CUDA_ERROR(detail::cuda_stream_create(&fake_stream)); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_stream(handle, fake_stream)); + } else { + fake_stream = original_stream; + } + + cudaGraph_t graph_obj; + + cudaStreamCaptureStatus capture_status; + + _CUDNN_CHECK_CUDA_ERROR(detail::cuda_stream_is_capturing(fake_stream, &capture_status)); + + CUDNN_FE_LOG_LABEL_ENDL("INFO: capture_status " + << capture_status << " original_stream " + << ((original_stream == nullptr) ? "DEFAULT (NULL) Stream" : "NON-DEFAULT Stream")); + + if (capture_status != cudaStreamCaptureStatusNone) { + CUDNN_FE_LOG_LABEL_ENDL("INFO: cuda graph capture active, aborting warmup"); + return {error_code_t::OK, "cuda graph capture active, aborting warmup"}; + } + + _CUDNN_CHECK_CUDA_ERROR(detail::cuda_graph_begin_capture(fake_stream, cudaStreamCaptureModeRelaxed)); + + std::unordered_map tensor_uid_to_pointer_map; + + void *tmp_pointer = reinterpret_cast(0x7f0000000000llu); + + _CUDNN_CHECK_CUDA_ERROR(detail::cuda_malloc((void **)&tmp_pointer, 1024 * 1024)); + + float tmp_double = 1.0f; + void *cpu_pointer = reinterpret_cast(&tmp_double); + + for (auto const &tensor : deserialized_tensor_properties) { + if (tensor->get_is_virtual() == false) { + if (tensor->get_is_pass_by_value() == false) { + tensor_uid_to_pointer_map.emplace(tensor->get_uid(), tmp_pointer); + } else { + tensor_uid_to_pointer_map.emplace(tensor->get_uid(), cpu_pointer); + } + } + } + + CUDNN_FE_LOG_LABEL_ENDL("INFO: full_graph_inputs: " << full_graph_inputs.size() << " elements"); + for (auto const &tensor : full_graph_inputs) { + CUDNN_FE_LOG_LABEL_ENDL("\tuid: " << tensor->get_uid() + << ", is_pass_by_value = " << tensor->get_is_pass_by_value()); + if (tensor->get_is_pass_by_value() == false) { + tensor_uid_to_pointer_map.emplace(tensor->get_uid(), tmp_pointer); + } else { + tensor_uid_to_pointer_map.emplace(tensor->get_uid(), cpu_pointer); + } + } + CUDNN_FE_LOG_LABEL_ENDL("INFO: full_graph_outputs: " << full_graph_outputs.size() << " elements"); + for (auto const &tensor : full_graph_outputs) { + CUDNN_FE_LOG_LABEL_ENDL("\tuid: " << tensor->get_uid()); + tensor_uid_to_pointer_map.emplace(tensor->get_uid(), tmp_pointer); + } + + CHECK_CUDNN_FRONTEND_ERROR( + extend_tensor_map_with_pass_by_value_tensors_(tensor_uid_to_pointer_map, deserialized_pass_by_value)); + + auto cudnn_status = execute(handle, tensor_uid_to_pointer_map, tmp_pointer); + (void)cudnn_status; // No need to check bad executes + + _CUDNN_CHECK_CUDA_ERROR(detail::cuda_graph_end_capture(fake_stream, &graph_obj)); + + _CUDNN_CHECK_CUDA_ERROR(detail::cuda_graph_destroy(graph_obj)); + + _CUDNN_CHECK_CUDA_ERROR(detail::cuda_free(tmp_pointer)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_stream(handle, original_stream)); + + if (original_stream == nullptr) { + _CUDNN_CHECK_CUDA_ERROR(detail::cuda_stream_destroy(fake_stream)); + } + + CUDNN_FE_LOG_BANNER("WARMUP (END FAKE GRAPH CAPTURE) "); + + return {error_code_t::OK, ""}; + } + + error_t + serialize(std::vector &data) const { + CUDNN_FE_LOG_BANNER(" SERIALIZE PLAN "); +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + json j; + serialize(j); + + auto const candidate = plans.candidate; + auto execution_plan = plans.execution_plans[candidate]; + if (execution_plan != nullptr) { + auto serialized_plan = execution_plan->getJsonRepresentation(); + j["cudnn_backend_data"] = serialized_plan; + j["variant_pack_uids"] = variant_pack_uids; + } + + j["behavior_notes"] = plans.behavior_notes; + + std::unordered_map tensor_to_pass_by_value; + CHECK_CUDNN_FRONTEND_ERROR(collect_pass_by_value_tensors_subtree(tensor_to_pass_by_value)); + + // Convert pass_by_values to JSON (unordered_map with numeric keys needs manual conversion) + json pass_by_values_json = json::object(); + for (const auto &[uid, variant_value] : tensor_to_pass_by_value) { + json variant_json; + variant_json = variant_value; + pass_by_values_json[std::to_string(uid)] = variant_json; + } + j["pass_by_values"] = pass_by_values_json; + + std::unordered_map>> workspace_modifications; + int64_t workspace_offset = 0; + CHECK_CUDNN_FRONTEND_ERROR(collect_tensors_in_workspace_subtree(workspace_modifications, workspace_offset)); + + // Convert workspace_modifications to JSON (nlohmann::json doesn't support std::tuple directly) + json workspace_modifications_json = json::object(); + for (const auto &[uid, tuple_value] : workspace_modifications) { + json tuple_json = json::array(); + tuple_json.push_back(std::get<0>(tuple_value)); + tuple_json.push_back(std::get<1>(tuple_value)); + tuple_json.push_back(std::get<2>(tuple_value)); + workspace_modifications_json[std::to_string(uid)] = tuple_json; + } + j["workspace_modifications"] = workspace_modifications_json; + + j["variant_pack_replacements"] = variant_pack_replacements; + + j["fe_workspace_size"] = fe_workspace_size; + + std::vector> tensors_to_dump_uids; + for (auto const &[tensor, fmt] : tensors_to_dump) { + tensors_to_dump_uids.emplace_back(tensor->get_uid(), fmt); + } + j["tensors_to_dump"] = tensors_to_dump_uids; + + data = json::to_ubjson(j); + CUDNN_FE_LOG_BANNER(" SERIALIZE PLAN (ALL OK) "); + return {error_code_t::OK, ""}; +#else + CUDNN_FRONTEND_UNUSED(data); + return {error_code_t::GRAPH_NOT_SUPPORTED, "unavailable when compiled with CUDNN_FRONTEND_SKIP_JSON_LIB"}; +#endif + } + + /** + * @brief Deserialize an execution plan from a serialized byte blob. + * + * Parses @p data with from_ubjson and delegates to the json overload. Callers that + * have already parsed the blob should call the json overload directly to avoid a + * second parse. + * + * @param handle cuDNN handle used to rebuild the execution plan. + * @param data UBJSON blob previously produced by serialize(). + * @param enforce_precompiled When true, fail unless the blob carries a precompiled plan. + * @param run_warmup When false, skip the throwaway warmup capture. + * @return error_t OK on success, otherwise an error code describing the failure. + */ + error_t + deserialize(cudnnHandle_t handle, + std::vector const &data, + bool const enforce_precompiled = false, + bool run_warmup = true) { +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + return deserialize(handle, json::from_ubjson(data), enforce_precompiled, run_warmup); +#else + CUDNN_FRONTEND_UNUSED(handle); + CUDNN_FRONTEND_UNUSED(data); + CUDNN_FRONTEND_UNUSED(enforce_precompiled); + CUDNN_FRONTEND_UNUSED(run_warmup); + return {error_code_t::GRAPH_NOT_SUPPORTED, "unavailable when compiled with CUDNN_FRONTEND_SKIP_JSON_LIB"}; +#endif + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + /** + * @brief Deserialize an execution plan from an already-parsed json. + * + * Avoids a second from_ubjson parse. run_warmup=false skips the throwaway warmup capture. + * + * @param handle cuDNN handle used to rebuild the execution plan. + * @param j Parsed json graph, as produced by serialize(). + * @param enforce_precompiled When true, fail unless the json carries a precompiled plan. + * @param run_warmup When false, skip the throwaway warmup capture. + * @return error_t OK on success, otherwise an error code describing the failure. + */ + error_t + deserialize(cudnnHandle_t handle, json const &j, bool const enforce_precompiled = false, bool run_warmup = true) { + CUDNN_FE_LOG_BANNER(" DESERIALIZE PLAN WITH HANDLE "); + + // Clear deserialize-owned containers so a re-deserialize on the same Graph + // does not feed prepare_variant_pack_template() with stale entries from a + // prior deserialize(handle, old_data). + deserialized_tensor_properties.clear(); + deserialized_pass_by_value.clear(); + deserialized_workspace_modifications.clear(); + tensors_to_dump.clear(); + + if (j.contains("graph_uid") && !j["graph_uid"].is_null()) { + graph_uid = j["graph_uid"].get(); + } + + // Resolve tensor UIDs with deserialized_tensor_properties. + if (j.contains("tensors")) { + auto tensor_map = j["tensors"].get>(); + for (const auto &tensor_info : tensor_map) { + auto tensor_attributes = std::make_shared(); + from_json(tensor_info.second, *tensor_attributes); + deserialized_tensor_properties.insert(tensor_attributes); + } + } + + RETURN_CUDNN_FRONTEND_ERROR_IF( + enforce_precompiled && !j.contains("cudnn_backend_data"), + error_code_t::GRAPH_EXECUTION_PLAN_CREATION_FAILED, + "enforce_precompiled requested, but serialized graph has no precompiled execution plan"); + + auto serialized_plan = j["cudnn_backend_data"]; + CHECK_CUDNN_FRONTEND_ERROR(plans.build_plans(handle, serialized_plan)); + + plans.behavior_notes = j["behavior_notes"].get>>(); + + variant_pack_uids = j["variant_pack_uids"].get>(); + + // Deserialize pass_by_values from JSON + if (j.contains("pass_by_values")) { + auto pass_by_values_json = j["pass_by_values"]; + for (auto it = pass_by_values_json.begin(); it != pass_by_values_json.end(); ++it) { + uid_t uid = std::stoll(it.key()); + pass_by_values_t value = it.value().get(); + deserialized_pass_by_value[uid] = value; + } + } + + // Deserialize workspace_modifications from JSON + if (j.contains("workspace_modifications")) { + auto workspace_modifications_json = j["workspace_modifications"]; + for (auto it = workspace_modifications_json.begin(); it != workspace_modifications_json.end(); ++it) { + uid_t uid = std::stoll(it.key()); + auto tuple_json = it.value(); + auto tuple_value = std::make_tuple(tuple_json[0].get(), + tuple_json[1].get(), + tuple_json[2].get>()); + deserialized_workspace_modifications[uid] = tuple_value; + } + } + + variant_pack_replacements = j["variant_pack_replacements"]; + + fe_workspace_size = j["fe_workspace_size"]; + + // Initialize the execution caches from deserialized data + cached_pass_by_value = deserialized_pass_by_value; + cached_workspace_modifications = deserialized_workspace_modifications; + + // Reset prep state in case this Graph is being re-deserialized; otherwise the + // eager prep below would early-return with the old slot layout. + varpack_prep_state->prepared.store(false, std::memory_order_release); + varpack_template = {}; + + // Eager prep, matching what build_plans() does for fresh-build graphs. + CHECK_CUDNN_FRONTEND_ERROR(prepare_variant_pack_template()); + + if (j.contains("tensors_to_dump")) { + auto dump_uids = j["tensors_to_dump"].get>>(); + for (auto const &[uid, fmt] : dump_uids) { + for (auto const &tensor : deserialized_tensor_properties) { + if (tensor->get_uid() == uid) { + tensors_to_dump.emplace_back(tensor, fmt); + break; + } + } + } + } + + if (run_warmup) { + CHECK_CUDNN_FRONTEND_ERROR(warmup(handle)); + } + + CUDNN_FE_LOG_BANNER(" DESERIALIZE PLAN WITH HANDLE (ALL OK) "); + + return {error_code_t::OK, ""}; + } +#endif + + Type + getType() override { + return Type::COMPOSITE; + } + + Graph & + set_intermediate_data_type(DataType_t type); + Graph & + set_io_data_type(DataType_t type); + Graph & + set_compute_data_type(DataType_t type); + Graph & + set_dynamic_shape_enabled(bool is_enabled); + Graph & + set_override_shape_enabled(bool is_enabled); + Graph & + set_sm_count(int32_t type); + Graph & + set_sm_version(int32_t version); + Graph & + set_kernel_cache(std::shared_ptr cache); + Graph & + set_device_properties(std::shared_ptr device_prop); + + Graph & + set_name(std::string const &name) { + context.set_name(name); + return *this; + } + + error_t + query_tensor_attributes_of_uid(int64_t const uid, Tensor_attributes &tensor) const; + + std::shared_ptr + tensor(Tensor_attributes const &tensor); + + // Overloaded tensor() methods for compile-time constants + std::shared_ptr + tensor(float const &scalar, ScalarType scalar_type); + + std::shared_ptr + tensor(half const &scalar, ScalarType scalar_type); + + std::shared_ptr + tensor(nv_bfloat16 const &scalar, ScalarType scalar_type); + + std::shared_ptr + tensor(int32_t const &scalar, ScalarType scalar_type); + + std::shared_ptr + tensor(int64_t const &scalar, ScalarType scalar_type); + + std::shared_ptr + tensor(double const &scalar, ScalarType scalar_type); + + std::shared_ptr + tensor_like(std::shared_ptr const &tensor, std::string const &name = std::string{}); + + std::array, 3> layernorm(std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + Layernorm_attributes); + + std::array, 3> adalayernorm(std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + AdaLayernorm_attributes); + + std::array, 3> instancenorm(std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + Instancenorm_attributes); + + std::array, 5> batchnorm(std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + Batchnorm_attributes); + + std::shared_ptr batchnorm_inference(std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + Batchnorm_inference_attributes); + + std::array, 6> bn_finalize(std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + BN_finalize_attributes); + + std::shared_ptr conv_fprop(std::shared_ptr, + std::shared_ptr, + Conv_fprop_attributes); + + std::shared_ptr conv_dgrad(std::shared_ptr, + std::shared_ptr, + Conv_dgrad_attributes); + + std::shared_ptr conv_wgrad(std::shared_ptr, + std::shared_ptr, + Conv_wgrad_attributes); + + std::array, 5> dbn_weight(std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + DBN_weight_attributes); + + std::array, 3> batchnorm_backward(std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + Batchnorm_backward_attributes); + + std::array, 3> layernorm_backward(std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + Layernorm_backward_attributes); + + std::array, 3> adalayernorm_backward(std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + AdaLayernorm_backward_attributes); + + std::array, 3> instancenorm_backward(std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + Instancenorm_backward_attributes); + std::array, 2> genstats(std::shared_ptr, Genstats_attributes); + + std::array, 2> rmsnorm(std::shared_ptr, + std::shared_ptr, + Rmsnorm_attributes); + + std::shared_ptr rope(std::shared_ptr, + std::shared_ptr, + RoPE_attributes); + + std::shared_ptr rope_backward(std::shared_ptr, + std::shared_ptr, + RoPE_backward_attributes); + + std::array, 3> rmsnorm_backward(std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + Rmsnorm_backward_attributes); + + std::array, 2> sdpa(std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + SDPA_attributes); + + // FP8 version + std::array, 4> sdpa_fp8(std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + SDPA_fp8_attributes); + + // MXFP8 version + std::array, 3> sdpa_fp8(std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + SDPA_fp8_attributes); + + inline std::array, 7> sdpa_fp8_backward(std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + SDPA_fp8_backward_attributes); + + // MXFP8 version + std::array, 6> sdpa_fp8_backward(std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + SDPA_fp8_backward_attributes); + + std::array, 3> sdpa_backward(std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + SDPA_backward_attributes); + + std::shared_ptr slice(std::shared_ptr, Slice_attributes); + + std::shared_ptr transpose(std::shared_ptr, Transpose_attributes); + + std::array, 2> block_scale_quantize(std::shared_ptr, + Block_scale_quantize_attributes); + + std::shared_ptr block_scale_dequantize(std::shared_ptr, + std::shared_ptr, + Block_scale_dequantize_attributes); + + std::shared_ptr concatenate(std::vector>, + Concatenate_attributes); + + std::shared_ptr moe_grouped_matmul(std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + Moe_grouped_matmul_attributes); + + std::shared_ptr moe_grouped_matmul_bwd(std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + Moe_grouped_matmul_bwd_attributes); + + [[deprecated]] std::array, 2> + scaled_dot_product_flash_attention(std::shared_ptr q, + std::shared_ptr k, + std::shared_ptr v, + SDPA_attributes attributes) { + return sdpa(q, k, v, attributes); + } + [[deprecated]] std::array, 3> + scaled_dot_product_flash_attention_backward(std::shared_ptr q, + std::shared_ptr k, + std::shared_ptr v, + std::shared_ptr o, + std::shared_ptr dO, + std::shared_ptr stats, + SDPA_backward_attributes attributes) { + return sdpa_backward(q, k, v, o, dO, stats, attributes); + } + + error_t + create_execution_plans(std::vector const &mode); + + error_t + create_execution_plan(int64_t const engine_id, std::unordered_map const &knobs); + + int64_t + get_execution_plan_count() const; + + inline error_t + get_engine_count(int64_t &count); + + inline error_t + get_knobs_for_engine(int64_t const engine, std::vector &); + + error_t + check_support(cudnnHandle_t h) { + // handle not required anymore + // TODO: remove this function in next release + (void)h; + return check_support(); + } + + // overload for deviceless AoT compilation + error_t + check_support() { + // Check OSS engine first if registered + + CHECK_CUDNN_FRONTEND_ERROR(context.populate_sm_version_from_device()); + auto sm_version = context.get_sm_version(); + + // Check OSS SDPA engine + if (plans.has_oss_sdpa_engine()) { + auto oss_status = plans.check_oss_sdpa_engine_support(sm_version); + if (oss_status.is_good()) { + return {error_code_t::OK, ""}; + } + // Fall through to check other engines + } + + // Check OSS RmsNorm+SiLU engine + if (plans.has_oss_rms_norm_silu_engine()) { + auto oss_status = plans.check_oss_rms_norm_silu_support(sm_version); + if (oss_status.is_good()) { + return {error_code_t::OK, ""}; + } + // Fall through to check cuDNN plans + } + + CHECK_CUDNN_FRONTEND_ERROR(plans.check_support()); + return {error_code_t::OK, ""}; + } + + // TODO: remove this function in next release + error_t + build(cudnnHandle_t const &handle, + std::vector const &mode, + BuildPlanPolicy_t const policy = BuildPlanPolicy_t::HEURISTICS_CHOICE, + bool const do_multithreaded_builds = false); + + // overload for deviceless AoT compilation + error_t + build(std::vector const &mode, + BuildPlanPolicy_t const policy = BuildPlanPolicy_t::HEURISTICS_CHOICE, + bool const do_multithreaded_builds = false); + + // ----------------------------------------------------------------------- + // Variant pack template: pre-indexed flat array built at build_plans() time. + // Eliminates per-call map operations in execute(). + // ----------------------------------------------------------------------- + + struct VariantPackTemplate { + std::vector all_uids; // all UIDs in sorted order (user + internal) + std::vector template_ptrs; // pre-filled: pass_by_value entries, nullptr for user/workspace slots + std::vector user_slots; // indices into template_ptrs that the user fills (sorted UID order) + + // Pre-computed slot indices for per-call updates (avoid linear scans at execute time) + std::vector> workspace_slots; // (slot, byte_offset) + std::vector>> + replacement_slots; // (dst_slot, (src_slot, offset)) — only used by Slice nodes + }; + + VariantPackTemplate varpack_template; + + // Per-Graph sync state for lazy varpack-template prep. Wrapped in a box so + // Graph stays default-copy/move-constructible (samples build Graphs into + // std::tuple by copy). + struct VarpackPrepState { + std::atomic prepared{false}; + std::mutex mu; + }; + struct VarpackPrepStateBox { + std::unique_ptr ptr = std::make_unique(); + VarpackPrepStateBox() = default; + VarpackPrepStateBox(VarpackPrepStateBox &&) noexcept = default; + VarpackPrepStateBox & + operator=(VarpackPrepStateBox &&) noexcept = default; + // Copy semantics: never copy the prepared flag. The cached template_ptrs + // store raw addresses into the source Graph's pass-by-value storage; a + // copied Graph must rebuild its own template on first use. + VarpackPrepStateBox(VarpackPrepStateBox const & /*other*/) : ptr(std::make_unique()) {} + VarpackPrepStateBox & + operator=(VarpackPrepStateBox const &other) { + if (this != &other) { + ptr = std::make_unique(); + } + return *this; + } + VarpackPrepState * + operator->() const { + return ptr.get(); + } + }; + mutable VarpackPrepStateBox varpack_prep_state; + + // Prepares the variant pack template. Called automatically at the end of build_plans(). + error_t + prepare_variant_pack_template() { + std::lock_guard lk(varpack_prep_state->mu); + if (varpack_prep_state->prepared.load(std::memory_order_relaxed)) { + return {error_code_t::OK, ""}; + } + + VariantPackTemplate t; + + // 1. Start with variant_pack_uids + any replacement source UIDs not already included. + // Replacement sources (e.g. slice input on cuDNN < 9.22 pointer-arithmetic fallback) may not + // be in variant_pack_uids; we still need a slot for the source pointer when replacements apply. + t.all_uids.assign(variant_pack_uids.begin(), variant_pack_uids.end()); + for (auto const &[from_uid, value] : variant_pack_replacements) { + if (variant_pack_uids.find(from_uid) == variant_pack_uids.end()) { + t.all_uids.push_back(from_uid); + } + } + std::sort(t.all_uids.begin(), t.all_uids.end()); + t.template_ptrs.assign(t.all_uids.size(), nullptr); + + // 2. Build UID → slot index + std::unordered_map uid_to_slot; + for (int i = 0; i < (int)t.all_uids.size(); i++) { + uid_to_slot[t.all_uids[i]] = i; + } + + // 3. Pre-fill pass_by_value entries (scalars like epsilon, alpha, beta) + std::unordered_map pbv_ptrs; + CHECK_CUDNN_FRONTEND_ERROR(extend_tensor_map_with_pass_by_value_tensors_(pbv_ptrs, cached_pass_by_value)); + for (auto const &[uid, ptr] : pbv_ptrs) { + auto it = uid_to_slot.find(uid); + if (it != uid_to_slot.end()) { + t.template_ptrs[it->second] = ptr; + } else { + int slot = (int)t.all_uids.size(); + t.all_uids.push_back(uid); + t.template_ptrs.push_back(ptr); + uid_to_slot[uid] = slot; + } + } + + // 4. Register workspace entries (slots allocated, pointers filled at execute time) + for (auto const &[uid, data] : cached_workspace_modifications) { + auto it = uid_to_slot.find(uid); + if (it == uid_to_slot.end()) { + int slot = (int)t.all_uids.size(); + t.all_uids.push_back(uid); + t.template_ptrs.push_back(nullptr); // filled at execute time + uid_to_slot[uid] = slot; + } + } + + // 5. Identify user-fillable slots: nullptr entries that are NOT workspace/replacement targets + std::unordered_set workspace_uids; + for (auto const &[uid, data] : cached_workspace_modifications) { + workspace_uids.insert(uid); + } + std::unordered_set replacement_dst_uids; + for (auto const &[from_uid, value] : variant_pack_replacements) { + (void)from_uid; + replacement_dst_uids.insert(value.first); // value.first = to_uid (the destination) + } + for (int i = 0; i < (int)t.template_ptrs.size(); i++) { + int64_t uid = t.all_uids[i]; + if (t.template_ptrs[i] == nullptr && workspace_uids.find(uid) == workspace_uids.end() && + replacement_dst_uids.find(uid) == replacement_dst_uids.end()) { + t.user_slots.push_back(i); + } + } + + // 6. Pre-compute workspace slot indices + for (auto const &[uid, data] : cached_workspace_modifications) { + const auto &[operation_type, offset, vec_data] = data; + auto it = uid_to_slot.find(uid); + if (it != uid_to_slot.end()) { + t.workspace_slots.emplace_back(it->second, offset); + } + } + + // 7. Pre-compute replacement slot indices + // variant_pack_replacements: key = from_uid (source), value = {to_uid (destination), byte_offset} + // Meaning: ptr[to_uid] = ptr[from_uid] + byte_offset (e.g. legacy slice fallback on cuDNN < 9.22) + for (auto const &[from_uid, value] : variant_pack_replacements) { + const auto &[to_uid, byte_offset] = value; + auto it_src = uid_to_slot.find(from_uid); + auto it_dst = uid_to_slot.find(to_uid); + // Replacement: dst_ptr = src_ptr + byte_offset + if (it_src != uid_to_slot.end() && it_dst != uid_to_slot.end()) { + t.replacement_slots.emplace_back(it_dst->second, std::make_pair(it_src->second, byte_offset)); + } + } + + varpack_template = std::move(t); + // Apply OSS slot indices before the release-store so a reader that + // observes prepared=true also sees the slot writes. + apply_oss_slot_indices_to_plans(); + varpack_prep_state->prepared.store(true, std::memory_order_release); + + return {error_code_t::OK, ""}; + } + + // Depends on plans.candidate; kept separate so autotune can re-apply without + // rebuilding the (candidate-independent) variant pack template. + void + apply_oss_slot_indices_to_plans() { + plans.set_oss_slot_indices([this](int64_t uid) -> int { + if (uid < 0) return -1; + for (size_t i = 0; i < varpack_template.all_uids.size(); ++i) { + if (varpack_template.all_uids[i] == uid) return static_cast(i); + } + return -1; + }); + } + + std::vector + get_variant_pack_uids_sorted() const { + std::vector user_uids; + for (int slot : varpack_template.user_slots) { + user_uids.push_back(varpack_template.all_uids[slot]); + } + return user_uids; + } + + error_t + build_plans(cudnnHandle_t const &handle, + BuildPlanPolicy_t const policy = BuildPlanPolicy_t::HEURISTICS_CHOICE, + bool const do_multithreaded_builds = false) { + // handle not required anymore + // TODO: remove this function in next release + (void)handle; + return build_plans(policy, do_multithreaded_builds); + } + + // overload for deviceless AoT compilation + error_t + build_plans(BuildPlanPolicy_t const policy = BuildPlanPolicy_t::HEURISTICS_CHOICE, + bool const do_multithreaded_builds = false); + + error_t + build_plan_at_index(cudnnHandle_t const &handle, int64_t index) { + // handle not required anymore + // TODO: remove this function in next release + (void)handle; + return build_plan_at_index(index); + } + + // overload for deviceless AoT compilation + error_t + build_plan_at_index(int64_t index); + + Graph & + deselect_workspace_greater_than(int64_t const workspace) { + plans.set_max_workspace_allowed(workspace); + return *this; + } + + Graph & + deselect_shared_mem_greater_than(int64_t const workspace) { + plans.set_max_shared_mem_allowed(workspace); + return *this; + } + + Graph & + deselect_engines(std::vector const &engine_names) { + plans.set_barred_names(engine_names); + return *this; + } + + Graph & + select_behavior_notes(std::vector const ¬es) { + auto status = plans.filter_behavior_notes(notes, true); + if (status.is_bad()) { + CUDNN_FE_LOG(status.get_message() << std::endl); + } + return *this; + } + + Graph & + select_numeric_notes(std::vector const ¬es) { + auto status = plans.filter_numeric_notes(notes, true); + if (status.is_bad()) { + CUDNN_FE_LOG(status.get_message() << std::endl); + } + return *this; + } + + Graph & + deselect_behavior_notes(std::vector const ¬es) { + auto status = plans.filter_behavior_notes(notes, false); + if (status.is_bad()) { + CUDNN_FE_LOG(status.get_message() << std::endl); + } + return *this; + } + + Graph & + deselect_numeric_notes(std::vector const ¬es) { + auto status = plans.filter_numeric_notes(notes, false); + if (status.is_bad()) { + CUDNN_FE_LOG(status.get_message() << std::endl); + } + return *this; + } + + error_t + get_behavior_notes_for_plan_at_index(int64_t const index, std::vector ¬es) const; + + error_t + get_behavior_notes(std::vector ¬es) const; + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json &j) const override final { + // Different from serialization of other INodes. + // Go over each subnode and serialize them. + json full_json; + + full_json["context"]["name"] = context.get_name(); + full_json["context"]["compute_data_type"] = context.get_compute_data_type(); + full_json["context"]["intermediate_data_type"] = context.get_intermediate_data_type(); + full_json["context"]["io_data_type"] = context.get_io_data_type(); + full_json["context"]["sm_count"] = context.get_target_sm_count(); + full_json["context"]["is_dynamic_shape_enabled"] = context.get_dynamic_shape_enabled(); + full_json["context"]["is_override_shape_enabled"] = context.get_override_shape_enabled(); + full_json["graph_uid"] = graph_uid; + + full_json.update(R"( {"tag": "GRAPH"})"_json); + full_json["nodes"]; + for (auto const &sub_node : sub_nodes) { + json j_sub_node; + sub_node->serialize(j_sub_node); + full_json["nodes"].push_back(j_sub_node); + } + + j["context"] = full_json["context"]; + j["graph_uid"] = full_json["graph_uid"]; + + j["json_version"] = "1.0"; + j["cudnn_backend_version"] = detail::get_backend_version_string(); + j["cudnn_frontend_version"] = CUDNN_FRONTEND_VERSION; + j["nodes"]; + j["tensors"]; + std::unordered_set tensors; + for (const auto &sub_node : full_json["nodes"]) { + // Create a short version of the node + auto short_node = sub_node; + short_node["inputs"] = {}; + short_node["outputs"] = {}; + + auto node_name = sub_node["tag"].get(); + auto i = 0; + // Process node inputs + for (const auto &input : sub_node["inputs"]) { + std::string port_name; + json tensor_info; + + if (node_name == "CONCATENATE") { + // Extract port_name and tensor_name + port_name = std::to_string(i); + tensor_info = input; + i++; + } else { + // Extract port_name and tensor_name + port_name = input[0].get(); + tensor_info = input[1]; + } + + if (tensor_info.is_null()) { + continue; + } + + // Determine the key to use for this tensor + std::string tensor_key; + json tensor_ref; + bool uid_assigned = tensor_info.contains("uid_assigned") && tensor_info["uid_assigned"].get(); + + if (uid_assigned && tensor_info.contains("uid") && tensor_info["uid"].is_number_integer()) { + // Use numeric UID if it was explicitly assigned + int64_t tensor_uid = tensor_info["uid"].get(); + tensor_key = std::to_string(tensor_uid); + tensor_ref = json(tensor_uid); + } else if (tensor_info.contains("name")) { + // Fall back to tensor name if UID not assigned + tensor_key = tensor_info["name"].get(); + tensor_ref = tensor_key; + } else { + continue; + } + + // Update short_node inputs + short_node["inputs"][port_name] = tensor_ref; + + // Check if the tensor is already in the tensors map + if (tensors.find(tensor_key) == tensors.end()) { + // If not, add it to the j["tensors"] + j["tensors"][tensor_key] = tensor_info; + } + } + + // Process node outputs + for (const auto &output : sub_node["outputs"]) { + // Extract port_name and tensor_name + auto port_name = output[0].get(); + auto tensor_info = output[1]; + + if (tensor_info.is_null()) { + continue; + } + + // Determine the key to use for this tensor + std::string tensor_key; + json tensor_ref; + bool uid_assigned = tensor_info.contains("uid_assigned") && tensor_info["uid_assigned"].get(); + + if (uid_assigned && tensor_info.contains("uid") && tensor_info["uid"].is_number_integer()) { + // Use numeric UID if it was explicitly assigned + int64_t tensor_uid = tensor_info["uid"].get(); + tensor_key = std::to_string(tensor_uid); + tensor_ref = json(tensor_uid); + } else if (tensor_info.contains("name")) { + // Fall back to tensor name if UID not assigned + tensor_key = tensor_info["name"].get(); + tensor_ref = tensor_key; + } else { + continue; + } + + // Update short_node outputs + short_node["outputs"][port_name] = tensor_ref; + + // Check if the tensor is already in the tensors map + if (tensors.find(tensor_key) == tensors.end()) { + // If not, add it to the j["tensors"] + j["tensors"][tensor_key] = tensor_info; + } + } + + // Add the short_node to j["nodes"] + j["nodes"].push_back(short_node); + } + }; +#endif + + size_t + key() override final { + return key(context.get_dynamic_shape_enabled()); + } + + // TODO: temparorily placed in graphs class. This function needs to be a free standing function. +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + error_t + deserialize(const json &j, bool const enforce_precompiled = false) { + RETURN_CUDNN_FRONTEND_ERROR_IF( + enforce_precompiled, + error_code_t::GRAPH_NOT_SUPPORTED, + "enforce_precompiled requires plan serialization; JSON deserialization reconstructs the graph"); + + if (j.contains("context")) { + const auto &j_context = j["context"]; + if (j_context.contains("compute_data_type") && !j_context["compute_data_type"].is_null()) { + context.set_compute_data_type(j_context["compute_data_type"].get()); + } + if (j_context.contains("intermediate_data_type") && !j_context["intermediate_data_type"].is_null()) { + context.set_intermediate_data_type(j_context["intermediate_data_type"].get()); + } + if (j_context.contains("io_data_type") && !j_context["io_data_type"].is_null()) { + context.set_io_data_type(j_context["io_data_type"].get()); + } + if (j_context.contains("name") && !j_context["name"].is_null()) { + context.set_name(j_context["name"].get()); + } + if (j_context.contains("sm_count") && !j_context["sm_count"].is_null()) { + context.set_target_sm_count(j_context["sm_count"].get()); + } + if (j_context.contains("is_dynamic_shape_enabled") && !j_context["is_dynamic_shape_enabled"].is_null()) { + context.set_dynamic_shape_enabled(j_context["is_dynamic_shape_enabled"].get()); + } + if (j_context.contains("is_override_shape_enabled") && !j_context["is_override_shape_enabled"].is_null()) { + context.set_override_shape_enabled(j_context["is_override_shape_enabled"].get()); + } + } + + if (j.contains("graph_uid") && !j["graph_uid"].is_null()) { + graph_uid = j["graph_uid"].get(); + } + + std::map> created_tensors; + // Iterate through each sub-node in the full JSON + if (j.contains("nodes") && j["nodes"].is_array()) { + for (auto j_sub_node : j["nodes"]) { + // Create a JSON object for inputs + json inputs; + + // Iterate through each input of the sub-node + if (j_sub_node.contains("inputs") && j_sub_node["inputs"].is_object()) { + for (auto &[port_name, tensor_ref] : j_sub_node["inputs"].items()) { + // Convert tensor reference (either numeric UID or string name) to string key + std::string tensor_key = tensor_ref.is_number_integer() + ? std::to_string(tensor_ref.get()) + : tensor_ref.get(); + + if (j.contains("tensors") && j["tensors"].contains(tensor_key)) { + // Add the input to the inputs JSON object + inputs.push_back({port_name, j["tensors"][tensor_key]}); + } + } + } + + // Create a JSON object for outputs + json outputs; + + // Iterate through each output of the sub-node + if (j_sub_node.contains("outputs") && j_sub_node["outputs"].is_object()) { + for (auto &[port_name, tensor_ref] : j_sub_node["outputs"].items()) { + // Convert tensor reference (either numeric UID or string name) to string key + std::string tensor_key = tensor_ref.is_number_integer() + ? std::to_string(tensor_ref.get()) + : tensor_ref.get(); + + if (j.contains("tensors") && j["tensors"].contains(tensor_key)) { + // Add the output to the outputs JSON object + outputs.push_back({port_name, j["tensors"][tensor_key]}); + } + } + } + + // Replace the original inputs and outputs of the sub-node with the new JSON objects + j_sub_node["inputs"] = inputs; + j_sub_node["outputs"] = outputs; + + auto check_if_pre_created_tensor = [&created_tensors](std::shared_ptr t) { + if (t == nullptr) { + return t; + } + + if (created_tensors.find(t->get_name()) == created_tensors.end()) { + created_tensors.insert({t->get_name(), t}); + return t; + } else { + return created_tensors[t->get_name()]; + } + }; + +#define CHECK_TENSORS(attributes) \ + for (const auto &[key, tensor] : attributes.inputs) { \ + attributes.inputs[key] = check_if_pre_created_tensor(tensor); \ + } \ + for (const auto &[key, tensor] : attributes.outputs) { \ + attributes.outputs[key] = check_if_pre_created_tensor(tensor); \ + } + +#define FILL_GLOBAL_IO_TENSOR_MAP(attributes) \ + for (auto input_name_to_attr_pair : attributes.inputs) { \ + if (input_name_to_attr_pair.second != nullptr && \ + (input_name_to_attr_pair.second->get_is_virtual() == false)) { \ + full_graph_inputs.emplace(input_name_to_attr_pair.second); \ + } \ + } \ + for (auto output_name_to_attr_pair : attributes.outputs) { \ + if (output_name_to_attr_pair.second != nullptr) { \ + full_graph_outputs.emplace(output_name_to_attr_pair.second); \ + } \ + } + if (j_sub_node.contains("tag") && j_sub_node["tag"].is_string()) { + auto tag = j_sub_node["tag"].get(); + if (tag == "CONV_FPROP") { + auto conv_fprop_attributes = j_sub_node.get(); + CHECK_TENSORS(conv_fprop_attributes); + FILL_GLOBAL_IO_TENSOR_MAP(conv_fprop_attributes); + sub_nodes.emplace_back( + std::make_unique(std::move(conv_fprop_attributes), context)); + } else if (tag == "POINTWISE") { + auto pointwise_attributes = j_sub_node.get(); + CHECK_TENSORS(pointwise_attributes); + FILL_GLOBAL_IO_TENSOR_MAP(pointwise_attributes); + sub_nodes.emplace_back( + std::make_unique(std::move(pointwise_attributes), context)); + } else if (tag == "REDUCTION") { + auto reduction_attributes = j_sub_node.get(); + CHECK_TENSORS(reduction_attributes); + FILL_GLOBAL_IO_TENSOR_MAP(reduction_attributes); + sub_nodes.emplace_back( + std::make_unique(std::move(reduction_attributes), context)); + } else if (tag == "SDPA") { + auto sdpa_attributes = j_sub_node.get(); + CHECK_TENSORS(sdpa_attributes); + FILL_GLOBAL_IO_TENSOR_MAP(sdpa_attributes); + switch (sdpa_attributes.implementation) { + case AttentionImplementation_t::AUTO: + return {error_code_t::INVALID_VALUE, + "Implementation cannot be AUTO in serialized form"}; + case AttentionImplementation_t::COMPOSITE: + sub_nodes.emplace_back( + std::make_unique(std::move(sdpa_attributes), context)); + break; + case AttentionImplementation_t::UNIFIED: + sub_nodes.emplace_back( + std::make_unique(std::move(sdpa_attributes), context)); + } + } else if (tag == "SDPA_BWD") { + auto sdpa_bwd_attributes = j_sub_node.get(); + CHECK_TENSORS(sdpa_bwd_attributes); + FILL_GLOBAL_IO_TENSOR_MAP(sdpa_bwd_attributes); + sub_nodes.emplace_back( + std::make_unique(std::move(sdpa_bwd_attributes), context)); + } else if (tag == "MATMUL") { + auto matmul_attributes = j_sub_node.get(); + CHECK_TENSORS(matmul_attributes); + FILL_GLOBAL_IO_TENSOR_MAP(matmul_attributes); + sub_nodes.emplace_back(std::make_unique(std::move(matmul_attributes), context)); + } else if (tag == "SLICE") { + auto slice_attributes = j_sub_node.get(); + CHECK_TENSORS(slice_attributes); + FILL_GLOBAL_IO_TENSOR_MAP(slice_attributes); + sub_nodes.emplace_back(std::make_unique(std::move(slice_attributes), context)); + } else if (tag == "TRANSPOSE") { + auto transpose_attributes = j_sub_node.get(); + CHECK_TENSORS(transpose_attributes); + FILL_GLOBAL_IO_TENSOR_MAP(transpose_attributes); + sub_nodes.emplace_back( + std::make_unique(std::move(transpose_attributes), context)); + } else if (tag == "RESAMPLE") { + auto resample_attributes = j_sub_node.get(); + CHECK_TENSORS(resample_attributes); + FILL_GLOBAL_IO_TENSOR_MAP(resample_attributes); + sub_nodes.emplace_back(std::make_unique(std::move(resample_attributes), context)); + } else if (tag == "CONV_DGRAD") { + auto dgrad_attributes = j_sub_node.get(); + CHECK_TENSORS(dgrad_attributes); + FILL_GLOBAL_IO_TENSOR_MAP(dgrad_attributes); + sub_nodes.emplace_back(std::make_unique(std::move(dgrad_attributes), context)); + } else if (tag == "CONV_WGRAD") { + auto wgrad_attributes = j_sub_node.get(); + CHECK_TENSORS(wgrad_attributes); + FILL_GLOBAL_IO_TENSOR_MAP(wgrad_attributes); + sub_nodes.emplace_back(std::make_unique(std::move(wgrad_attributes), context)); + } else if (tag == "MOE_GROUPED_MATMUL") { + auto moe_grouped_matmul_attributes = j_sub_node.get(); + CHECK_TENSORS(moe_grouped_matmul_attributes); + FILL_GLOBAL_IO_TENSOR_MAP(moe_grouped_matmul_attributes); + sub_nodes.emplace_back( + std::make_unique(std::move(moe_grouped_matmul_attributes), context)); + } + } +#undef CHECK_TENSORS + } + } + + return {error_code_t::OK, ""}; + } +#endif + + std::string + print(void) const { +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + std::stringstream ss; + json j = *this; + ss << j; + return ss.str(); +#else + return "print is unavailable when compiled with CUDNN_FRONTEND_SKIP_JSON_LIB"; +#endif + } +}; + +inline error_t +Graph::get_behavior_notes_for_plan_at_index(int64_t const index, std::vector ¬es) const { + CHECK_CUDNN_FRONTEND_ERROR(plans.get_behavior_notes_at_index(index, notes)); + return {error_code_t::OK, ""}; +} + +inline error_t +Graph::get_behavior_notes(std::vector ¬es) const { + int64_t const candidate = plans.candidate; + RETURN_CUDNN_FRONTEND_ERROR_IF( + candidate == -1, + error_code_t::INVALID_VALUE, + "No candiate plan set for the graph. You can set one by building a plan, which in turn sets the " + "candidate internally. Do note that you also query behaviour notes for a created-but-not-built plan by using " + "get_behavior_notes_for_plan_at_index API."); + + CHECK_CUDNN_FRONTEND_ERROR(get_behavior_notes_for_plan_at_index(candidate, notes)); + return {error_code_t::OK, ""}; +} + +inline int64_t +Graph::get_execution_plan_count() const { + return plans.execution_plans.size(); +} + +inline error_t +Graph::get_engine_count(int64_t &count) { + _CUDNN_CHECK_CUDNN_ERROR(detail::get_attribute(operation_graph->get_raw_desc(), + CUDNN_ATTR_OPERATIONGRAPH_ENGINE_GLOBAL_COUNT, + CUDNN_TYPE_INT64, + 1, + nullptr, + &count)); + + return {error_code_t::OK, ""}; +} + +inline error_t +Graph::get_knobs_for_engine(int64_t const engine, std::vector &knobs) { + CHECK_CUDNN_FRONTEND_ERROR(detail::query_knobs(engine, operation_graph->get_raw_desc(), knobs)); + + return {error_code_t::OK, ""}; +} + +inline error_t +Graph::create_execution_plans(std::vector const &mode) { + CUDNN_FE_LOG_BANNER(" CREATE EXECUTION PLANS (HEURISTICS QUERY) "); + + // CHECK IF NEED TO OVERRIDE HEURISTICS QUERY + for (auto &sub_node : sub_nodes) { + if (auto [engine_id, user_knobs] = sub_node->override_heuristics_query(); engine_id != -1) { +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + CUDNN_FE_LOG_LABEL_ENDL("INFO: Overriding heuristics query with engine ID " + << engine_id << " and user knobs " << nlohmann::json(user_knobs).dump()); +#else + CUDNN_FE_LOG_LABEL_ENDL("INFO: Overriding heuristics query with engine ID " + << engine_id << " and user knobs " << static_cast(user_knobs.size())); +#endif + CHECK_CUDNN_FRONTEND_ERROR(create_execution_plan(engine_id, user_knobs)); + return {error_code_t::OK, ""}; + } + } + + // Separate OPENSOURCE from cuDNN modes + bool has_opensource = false; + std::vector cudnn_modes; + for (auto const &m : mode) { + if (m == HeurMode_t::OPENSOURCE) { + has_opensource = true; + } else { + cudnn_modes.push_back(m); + } + } + + // Register OSS engines if OPENSOURCE mode requested + if (has_opensource) { + // Try SDPA OSS engine + auto oss_sdpa_status = register_oss_engine_(); + if (oss_sdpa_status.is_good()) { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Registered OSS SDPA prefill engine"); + } + + // Try RmsNorm+SiLU OSS engine + auto oss_norm_status = register_oss_rms_norm_silu_engine_(); + if (oss_norm_status.is_good()) { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Registered OSS RmsNorm+SiLU engine"); + } + + if (oss_sdpa_status.is_bad() && oss_norm_status.is_bad()) { + CUDNN_FE_LOG_LABEL_ENDL("WARN: No OSS engine matched the graph pattern"); + } + } + + // Query cuDNN heuristics for non-OPENSOURCE modes + if (!cudnn_modes.empty()) { + EngineConfigList op_graph_to_configs; + CHECK_CUDNN_FRONTEND_ERROR(detail::query_cudnn_heuristics_impl( + operation_graph, op_graph_to_configs, cudnn_modes, context.get_target_sm_count(), device_properties)); + + CUDNN_FE_LOG_LABEL_ENDL("INFO: Extracting engine configs."); + + plans.set_tag(operation_graph->getTag()); + plans.enqueue_engine_configs(op_graph_to_configs); + plans.set_kernel_cache(kernel_cache); + + CUDNN_FE_LOG_LABEL_ENDL("INFO: Querying engine config properties."); + CHECK_CUDNN_FRONTEND_ERROR(plans.query_properties()); + } + + CUDNN_FE_LOG_BANNER(" HEURISTICS QUERY ALL OK "); + return {error_code_t::OK, ""}; +} + +inline error_t +Graph::create_execution_plan(int64_t const engine_id, std::unordered_map const &user_knobs) { + // first create the engine + // this just uses the global engine id and operation graph + CUDNN_FE_LOG_BANNER(" CREATE EXECUTION PLAN for engine id " << engine_id << " "); + detail::backend_descriptor engine(CUDNN_BACKEND_ENGINE_DESCRIPTOR); + RETURN_CUDNN_FRONTEND_ERROR_IF(engine.get_status() != CUDNN_STATUS_SUCCESS, + error_code_t::CUDNN_BACKEND_API_FAILED, + "Failed to create engine's backend descriptor."); + CHECK_CUDNN_FRONTEND_ERROR( + detail::create_engine(engine, engine_id, operation_graph->get_raw_desc(), device_properties)); + + // Create an array of knob choices + std::vector knob_choices; + CHECK_CUDNN_FRONTEND_ERROR(detail::set_knob_choices(user_knobs, knob_choices)); + + auto engine_config = make_shared_backend_pointer((cudnnBackendDescriptorType_t)CUDNN_BACKEND_ENGINECFG_DESCRIPTOR); + CHECK_CUDNN_FRONTEND_ERROR(detail::create_engine_config(engine_config, engine, knob_choices)); + plans.enqueue_engine_configs({engine_config}); + CHECK_CUDNN_FRONTEND_ERROR(plans.query_properties()); + + CUDNN_FE_LOG_BANNER(" CREATE EXECUTION PLAN ALL OK "); + + return {error_code_t::OK, ""}; +} + +inline error_t +Graph::build_plan_at_index(int64_t plan_index) { + CHECK_CUDNN_FRONTEND_ERROR(plans.build_plan_at_index(plan_index)); + return {error_code_t::OK, ""}; +} + +inline error_t +Graph::build_plans(BuildPlanPolicy_t const policy, bool const do_multithreaded_builds) { +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + CUDNN_FE_LOG_BANNER(" BUILD PLANS for policy " << nlohmann::json(policy).dump() << " "); +#else + CUDNN_FE_LOG_BANNER(" BUILD PLANS for policy " << static_cast(policy) << " "); +#endif + + // Build OSS SDPA engine if it passed check_support + if (plans.has_oss_sdpa_engine()) { + auto oss_status = plans.build_oss_sdpa_engine(); + if (oss_status.is_good()) { + CUDNN_FE_LOG_LABEL_ENDL("INFO: OSS SDPA engine built successfully (NVRTC compilation done)"); + if (policy == BuildPlanPolicy_t::HEURISTICS_CHOICE) { + CUDNN_FE_LOG_BANNER(" BUILD PLANS ALL OK (OSS SDPA engine) "); + return {error_code_t::OK, ""}; + } + } else { + CUDNN_FE_LOG_LABEL_ENDL("WARN: OSS SDPA engine build failed: " << oss_status.get_message()); + } + } + + // Build OSS RmsNorm+SiLU engine if it passed check_support + if (plans.has_oss_rms_norm_silu_engine()) { + auto oss_status = plans.build_oss_rms_norm_silu_engine(); + if (oss_status.is_good()) { + CUDNN_FE_LOG_LABEL_ENDL("INFO: OSS RmsNorm+SiLU engine built successfully (NVRTC compilation done)"); + if (policy == BuildPlanPolicy_t::HEURISTICS_CHOICE) { + CUDNN_FE_LOG_BANNER(" BUILD PLANS ALL OK (OSS RmsNorm+SiLU engine) "); + return {error_code_t::OK, ""}; + } + } else { + CUDNN_FE_LOG_LABEL_ENDL("WARN: OSS RmsNorm+SiLU engine build failed: " << oss_status.get_message()); + } + } + + CHECK_CUDNN_FRONTEND_ERROR(plans.build_plans(policy, do_multithreaded_builds)); + + // Prepare the variant pack template for fast execution. + // This pre-computes slot indices so execute() can skip map operations. + CHECK_CUDNN_FRONTEND_ERROR(prepare_variant_pack_template()); + + CUDNN_FE_LOG_BANNER(" BUILD PLANS ALL OK "); + return {error_code_t::OK, ""}; +} + +inline error_t +Graph::build(cudnnHandle_t const &handle, + std::vector const &modes, + BuildPlanPolicy_t const policy, + bool const do_multithreaded_builds) { +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + CUDNN_FE_LOG_BANNER(" BUILD with handle " << nlohmann::json(policy).dump()); +#else + CUDNN_FE_LOG_BANNER(" BUILD with handle " << static_cast(policy) << " "); +#endif + CHECK_CUDNN_FRONTEND_ERROR(this->validate()); + CHECK_CUDNN_FRONTEND_ERROR(this->build_operation_graph(handle)); + CHECK_CUDNN_FRONTEND_ERROR(this->create_execution_plans(modes)); + CHECK_CUDNN_FRONTEND_ERROR(this->check_support()); + CHECK_CUDNN_FRONTEND_ERROR(this->build_plans(policy, do_multithreaded_builds)); + CUDNN_FE_LOG_BANNER(" BUILD ALL OK (with handle) "); + return {error_code_t::OK, ""}; +} + +inline error_t +Graph::build(std::vector const &modes, BuildPlanPolicy_t const policy, bool const do_multithreaded_builds) { +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + CUDNN_FE_LOG_BANNER(" BUILD PLANS without handle " << nlohmann::json(policy).dump() << " "); +#else + CUDNN_FE_LOG_BANNER(" BUILD PLANS without handle " << static_cast(policy) << " "); +#endif + CHECK_CUDNN_FRONTEND_ERROR(this->validate()); + CHECK_CUDNN_FRONTEND_ERROR(this->build_operation_graph()); + CHECK_CUDNN_FRONTEND_ERROR(this->create_execution_plans(modes)); + CHECK_CUDNN_FRONTEND_ERROR(this->check_support()); + CHECK_CUDNN_FRONTEND_ERROR(this->build_plans(policy, do_multithreaded_builds)); + CUDNN_FE_LOG_BANNER(" BUILD PLANS ALL OK (no handle) "); + return {error_code_t::OK, ""}; +} + +inline Graph & +Graph::set_intermediate_data_type(DataType_t const type) { + context.set_intermediate_data_type(type); + return *this; +} + +inline Graph & +Graph::set_io_data_type(DataType_t const type) { + context.set_io_data_type(type); + return *this; +} + +inline Graph & +Graph::set_compute_data_type(DataType_t const type) { + context.set_compute_data_type(type); + return *this; +} + +inline Graph & +Graph::set_dynamic_shape_enabled(bool is_enabled) { + context.set_dynamic_shape_enabled(is_enabled); + this->is_dynamic_shape_enabled = is_enabled; + return *this; +} + +inline Graph & +Graph::set_override_shape_enabled(bool is_enabled) { + context.set_override_shape_enabled(is_enabled); + this->is_override_shape_enabled = is_enabled; + return *this; +} + +inline Graph & +Graph::set_kernel_cache(std::shared_ptr cache) { + kernel_cache = cache; + return *this; +} + +inline Graph & +Graph::set_device_properties(std::shared_ptr device_prop) { + device_properties = device_prop; + return *this; +} + +inline Graph & +Graph::set_sm_count(int32_t count) { + context.set_target_sm_count(count); + return *this; +} + +inline Graph & +Graph::set_sm_version(int32_t version) { + context.set_sm_version(version); + return *this; +} + +inline std::shared_ptr +Graph::tensor(Tensor_attributes const &tensor) { + auto tensor_ptr = std::make_shared(tensor); + full_graph_inputs.emplace(tensor_ptr); + return tensor_ptr; +} + +// Overloaded tensor() methods for compile-time constants +inline std::shared_ptr +Graph::tensor(float const &scalar, ScalarType scalar_type) { + auto tensor_ptr = std::make_shared(scalar, scalar_type); + full_graph_inputs.emplace(tensor_ptr); + return tensor_ptr; +} + +inline std::shared_ptr +Graph::tensor(half const &scalar, ScalarType scalar_type) { + auto tensor_ptr = std::make_shared(scalar, scalar_type); + full_graph_inputs.emplace(tensor_ptr); + return tensor_ptr; +} + +inline std::shared_ptr +Graph::tensor(nv_bfloat16 const &scalar, ScalarType scalar_type) { + auto tensor_ptr = std::make_shared(scalar, scalar_type); + full_graph_inputs.emplace(tensor_ptr); + return tensor_ptr; +} + +inline std::shared_ptr +Graph::tensor(int32_t const &scalar, ScalarType scalar_type) { + auto tensor_ptr = std::make_shared(scalar, scalar_type); + full_graph_inputs.emplace(tensor_ptr); + return tensor_ptr; +} + +inline std::shared_ptr +Graph::tensor(int64_t const &scalar, ScalarType scalar_type) { + auto tensor_ptr = std::make_shared(scalar, scalar_type); + full_graph_inputs.emplace(tensor_ptr); + return tensor_ptr; +} + +inline std::shared_ptr +Graph::tensor(double const &scalar, ScalarType scalar_type) { + auto tensor_ptr = std::make_shared(scalar, scalar_type); + full_graph_inputs.emplace(tensor_ptr); + return tensor_ptr; +} + +inline error_t +Graph::query_tensor_attributes_of_uid(int64_t const uid, Tensor_attributes &tensor) const { + for (auto const &o_tensor : full_graph_outputs) { + if (uid == o_tensor->get_uid()) { + tensor = *o_tensor; + return {error_code_t::OK, ""}; + } + } + + for (auto const &i_tensor : full_graph_inputs) { + if (uid == i_tensor->get_uid()) { + tensor = *i_tensor; + return {error_code_t::OK, ""}; + } + } + + for (auto const &d_tensor : deserialized_tensor_properties) { + if (uid == d_tensor->get_uid()) { + tensor = *d_tensor; + return {error_code_t::OK, ""}; + } + } + + return {error_code_t::INVALID_VALUE, "No matching tensor for this UID"}; +} + +// tensor_like is meant to create "useable" copies of a tensor. +// By usable, it means not copying over the uids, as uids are FE-level(internal) detail. +// It also means not copying over names, which are user-level(external) detail. But user is given option to provide a +// new name. +inline std::shared_ptr +Graph::tensor_like(std::shared_ptr const &tensor, std::string const &name) { + auto tensor_ptr = std::make_shared(*tensor); + + // reset the uid of the cloned tensor + // uids are not meant to be copied by tensor_like + // When lowering to cudnn backend, both tensors involved here will get unique uids. + tensor_ptr->clear_uid(); + + // reset the name too. Defaults to empty string. + tensor_ptr->set_name(name); + full_graph_inputs.emplace(tensor_ptr); + + return tensor_ptr; +} + +inline std::array, 6> +Graph::bn_finalize(std::shared_ptr sum, + std::shared_ptr sq_sum, + std::shared_ptr scale, + std::shared_ptr bias, + std::shared_ptr epsilon, + std::shared_ptr accum_count, + BN_finalize_attributes attributes) { + // Set outputs + auto EQ_SCALE = attributes.outputs[BN_finalize_attributes::output_names::EQ_SCALE] = + output_tensor(attributes.name + "::EQ_SCALE"); + auto EQ_BIAS = attributes.outputs[BN_finalize_attributes::output_names::EQ_BIAS] = + output_tensor(attributes.name + "::EQ_BIAS"); + auto MEAN = attributes.outputs[BN_finalize_attributes::output_names::MEAN] = + output_tensor(attributes.name + "::MEAN"); + auto INV_VARIANCE = attributes.outputs[BN_finalize_attributes::output_names::INV_VARIANCE] = + output_tensor(attributes.name + "::INV_VARIANCE"); + std::shared_ptr NEXT_RUNNING_MEAN = nullptr; + std::shared_ptr NEXT_RUNNING_VAR = nullptr; + if (attributes.inputs[BN_finalize_attributes::input_names::PREV_RUNNING_MEAN] && + attributes.inputs[BN_finalize_attributes::input_names::PREV_RUNNING_VAR] && + attributes.inputs[BN_finalize_attributes::input_names::MOMENTUM]) { + NEXT_RUNNING_MEAN = output_tensor(attributes.name + "::NEXT_RUNNING_MEAN"); + NEXT_RUNNING_VAR = output_tensor(attributes.name + "::NEXT_RUNNING_VAR"); + } + attributes.outputs[BN_finalize_attributes::output_names::NEXT_RUNNING_MEAN] = NEXT_RUNNING_MEAN; + attributes.outputs[BN_finalize_attributes::output_names::NEXT_RUNNING_VAR] = NEXT_RUNNING_VAR; + + // Set inputs + attributes.inputs[BN_finalize_attributes::input_names::SUM] = sum; + attributes.inputs[BN_finalize_attributes::input_names::SQ_SUM] = sq_sum; + attributes.inputs[BN_finalize_attributes::input_names::SCALE] = scale; + attributes.inputs[BN_finalize_attributes::input_names::BIAS] = bias; + attributes.inputs[BN_finalize_attributes::input_names::EPSILON] = epsilon; + attributes.inputs[BN_finalize_attributes::input_names::ACCUM_COUNT] = accum_count; + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + + return {EQ_SCALE, EQ_BIAS, MEAN, INV_VARIANCE, NEXT_RUNNING_MEAN, NEXT_RUNNING_VAR}; +} + +inline std::array, 3> +Graph::layernorm(std::shared_ptr x, + std::shared_ptr scale, + std::shared_ptr bias, + Layernorm_attributes attributes) { + // Set outputs + auto Y = attributes.outputs[Layernorm_attributes::output_names::Y] = output_tensor(attributes.name + "::Y"); + std::shared_ptr MEAN = nullptr; + std::shared_ptr INV_VARIANCE = nullptr; + if (attributes.forward_phase == NormFwdPhase_t::TRAINING) { + MEAN = attributes.outputs[Layernorm_attributes::output_names::MEAN] = output_tensor(attributes.name + "::MEAN"); + INV_VARIANCE = attributes.outputs[Layernorm_attributes::output_names::INV_VARIANCE] = + output_tensor(attributes.name + "::INV_VARIANCE"); + } + // Set inputs + attributes.inputs[Layernorm_attributes::input_names::X] = x; + attributes.inputs[Layernorm_attributes::input_names::SCALE] = scale; + attributes.inputs[Layernorm_attributes::input_names::BIAS] = bias; + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + + return {Y, MEAN, INV_VARIANCE}; +} + +inline std::array, 3> +Graph::adalayernorm(std::shared_ptr x, + std::shared_ptr scale, + std::shared_ptr bias, + AdaLayernorm_attributes attributes) { + // Set outputs + auto Y = attributes.outputs[AdaLayernorm_attributes::output_names::Y] = output_tensor(attributes.name + "::Y"); + std::shared_ptr MEAN = nullptr; + std::shared_ptr INV_VARIANCE = nullptr; + if (attributes.forward_phase == NormFwdPhase_t::TRAINING) { + MEAN = attributes.outputs[AdaLayernorm_attributes::output_names::MEAN] = + output_tensor(attributes.name + "::MEAN"); + INV_VARIANCE = attributes.outputs[AdaLayernorm_attributes::output_names::INV_VARIANCE] = + output_tensor(attributes.name + "::INV_VARIANCE"); + } + // Set inputs + attributes.inputs[AdaLayernorm_attributes::input_names::X] = x; + attributes.inputs[AdaLayernorm_attributes::input_names::SCALE] = scale; + attributes.inputs[AdaLayernorm_attributes::input_names::BIAS] = bias; + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + + return {std::move(Y), std::move(MEAN), std::move(INV_VARIANCE)}; +} + +inline std::array, 3> +Graph::instancenorm(std::shared_ptr x, + std::shared_ptr scale, + std::shared_ptr bias, + Instancenorm_attributes attributes) { + // Set outputs + auto Y = attributes.outputs[Instancenorm_attributes::output_names::Y] = output_tensor(attributes.name + "::Y"); + std::shared_ptr MEAN = nullptr; + std::shared_ptr INV_VARIANCE = nullptr; + if (attributes.forward_phase == NormFwdPhase_t::TRAINING) { + MEAN = attributes.outputs[Instancenorm_attributes::output_names::MEAN] = + output_tensor(attributes.name + "::MEAN"); + INV_VARIANCE = attributes.outputs[Instancenorm_attributes::output_names::INV_VARIANCE] = + output_tensor(attributes.name + "::INV_VARIANCE"); + } + // Set inputs + attributes.inputs[Instancenorm_attributes::input_names::X] = x; + attributes.inputs[Instancenorm_attributes::input_names::SCALE] = scale; + attributes.inputs[Instancenorm_attributes::input_names::BIAS] = bias; + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + + return {Y, MEAN, INV_VARIANCE}; +} + +inline std::array, 5> +Graph::batchnorm(std::shared_ptr x, + std::shared_ptr scale, + std::shared_ptr bias, + Batchnorm_attributes attributes) { + // Set outputs + auto Y = attributes.outputs[Batchnorm_attributes::output_names::Y] = output_tensor(attributes.name + "::Y"); + auto MEAN = attributes.outputs[Batchnorm_attributes::output_names::MEAN] = + output_tensor(attributes.name + "::MEAN"); + auto INV_VARIANCE = attributes.outputs[Batchnorm_attributes::output_names::INV_VARIANCE] = + output_tensor(attributes.name + "::INV_VARIANCE"); + std::shared_ptr NEXT_RUNNING_MEAN = nullptr; + std::shared_ptr NEXT_RUNNING_VAR = nullptr; + if (attributes.inputs[Batchnorm_attributes::input_names::PREV_RUNNING_MEAN] && + attributes.inputs[Batchnorm_attributes::input_names::PREV_RUNNING_VAR] && + attributes.inputs[Batchnorm_attributes::input_names::MOMENTUM]) { + NEXT_RUNNING_MEAN = output_tensor(attributes.name + "::NEXT_RUNNING_MEAN"); + NEXT_RUNNING_VAR = output_tensor(attributes.name + "::NEXT_RUNNING_VAR"); + } + attributes.outputs[Batchnorm_attributes::output_names::NEXT_RUNNING_MEAN] = NEXT_RUNNING_MEAN; + attributes.outputs[Batchnorm_attributes::output_names::NEXT_RUNNING_VAR] = NEXT_RUNNING_VAR; + + // Set inputs + attributes.inputs[Batchnorm_attributes::input_names::X] = x; + attributes.inputs[Batchnorm_attributes::input_names::SCALE] = scale; + attributes.inputs[Batchnorm_attributes::input_names::BIAS] = bias; + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + + return {Y, MEAN, INV_VARIANCE, NEXT_RUNNING_MEAN, NEXT_RUNNING_VAR}; +} + +inline std::shared_ptr +Graph::batchnorm_inference(std::shared_ptr x, + std::shared_ptr mean, + std::shared_ptr inv_variance, + std::shared_ptr scale, + std::shared_ptr bias, + Batchnorm_inference_attributes attributes) { + // Set outputs + auto Y = attributes.outputs[Batchnorm_inference_attributes::output_names::Y] = + output_tensor(attributes.name + "::Y"); + + // Set inputs + attributes.inputs[Batchnorm_inference_attributes::input_names::X] = x; + attributes.inputs[Batchnorm_inference_attributes::input_names::MEAN] = mean; + attributes.inputs[Batchnorm_inference_attributes::input_names::INV_VARIANCE] = inv_variance; + attributes.inputs[Batchnorm_inference_attributes::input_names::SCALE] = scale; + attributes.inputs[Batchnorm_inference_attributes::input_names::BIAS] = bias; + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + + return Y; +} + +inline std::array, 3> +Graph::batchnorm_backward(std::shared_ptr dy, + std::shared_ptr x, + std::shared_ptr scale, + Batchnorm_backward_attributes attributes) { + // Set outputs + auto DX = attributes.outputs[Batchnorm_backward_attributes::output_names::DX] = + output_tensor(attributes.name + "::DX"); + auto DSCALE = attributes.outputs[Batchnorm_backward_attributes::output_names::DSCALE] = + output_tensor(attributes.name + "::DSCALE"); + auto DBIAS = attributes.outputs[Batchnorm_backward_attributes::output_names::DBIAS] = + output_tensor(attributes.name + "::DBIAS"); + + // Set inputs + attributes.inputs[Batchnorm_backward_attributes::input_names::DY] = dy; + attributes.inputs[Batchnorm_backward_attributes::input_names::X] = x; + attributes.inputs[Batchnorm_backward_attributes::input_names::SCALE] = scale; + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + + return {DX, DSCALE, DBIAS}; +} + +inline std::array, 3> +Graph::instancenorm_backward(std::shared_ptr dy, + std::shared_ptr x, + std::shared_ptr scale, + Instancenorm_backward_attributes attributes) { + // Set outputs + auto DX = attributes.outputs[Instancenorm_backward_attributes::output_names::DX] = + output_tensor(attributes.name + "::DX"); + auto DSCALE = attributes.outputs[Instancenorm_backward_attributes::output_names::DSCALE] = + output_tensor(attributes.name + "::DSCALE"); + auto DBIAS = attributes.outputs[Instancenorm_backward_attributes::output_names::DBIAS] = + output_tensor(attributes.name + "::DBIAS"); + + // Set inputs + attributes.inputs[Instancenorm_backward_attributes::input_names::DY] = dy; + attributes.inputs[Instancenorm_backward_attributes::input_names::X] = x; + attributes.inputs[Instancenorm_backward_attributes::input_names::SCALE] = scale; + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + + return {DX, DSCALE, DBIAS}; +} + +inline std::array, 3> +Graph::layernorm_backward(std::shared_ptr dy, + std::shared_ptr x, + std::shared_ptr scale, + Layernorm_backward_attributes attributes) { + // Set outputs + auto DX = attributes.outputs[Layernorm_backward_attributes::output_names::DX] = + output_tensor(attributes.name + "::DX"); + auto DSCALE = attributes.outputs[Layernorm_backward_attributes::output_names::DSCALE] = + output_tensor(attributes.name + "::DSCALE"); + auto DBIAS = attributes.outputs[Layernorm_backward_attributes::output_names::DBIAS] = + output_tensor(attributes.name + "::DBIAS"); + + // Set inputs + attributes.inputs[Layernorm_backward_attributes::input_names::DY] = dy; + attributes.inputs[Layernorm_backward_attributes::input_names::X] = x; + attributes.inputs[Layernorm_backward_attributes::input_names::SCALE] = scale; + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + + return {DX, DSCALE, DBIAS}; +} + +inline std::array, 3> +Graph::adalayernorm_backward(std::shared_ptr dy, + std::shared_ptr x, + std::shared_ptr scale, + AdaLayernorm_backward_attributes attributes) { + // Set outputs + auto DX = attributes.outputs[AdaLayernorm_backward_attributes::output_names::DX] = + output_tensor(attributes.name + "::DX"); + auto DSCALE = attributes.outputs[AdaLayernorm_backward_attributes::output_names::DSCALE] = + output_tensor(attributes.name + "::DSCALE"); + auto DBIAS = attributes.outputs[AdaLayernorm_backward_attributes::output_names::DBIAS] = + output_tensor(attributes.name + "::DBIAS"); + // Set inputs + attributes.inputs[AdaLayernorm_backward_attributes::input_names::DY] = dy; + attributes.inputs[AdaLayernorm_backward_attributes::input_names::X] = x; + attributes.inputs[AdaLayernorm_backward_attributes::input_names::SCALE] = scale; + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + + return {std::move(DX), std::move(DSCALE), std::move(DBIAS)}; +} + +inline std::shared_ptr +Graph::conv_fprop(std::shared_ptr x, + std::shared_ptr w, + Conv_fprop_attributes attributes) { + // Make required output tensors + if (attributes.name.empty()) { + attributes.name += std::to_string(sub_nodes.size()); + } + auto Y = output_tensor(attributes.name + "::Y"); + attributes.outputs[Conv_fprop_attributes::output_names::Y] = Y; + + // Set inputs + attributes.inputs[Conv_fprop_attributes::input_names::X] = x; + attributes.inputs[Conv_fprop_attributes::input_names::W] = w; + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + + return Y; +} + +inline std::array, 5> +Graph::dbn_weight(std::shared_ptr dy, + std::shared_ptr x, + std::shared_ptr mean, + std::shared_ptr inv_variance, + std::shared_ptr scale, + DBN_weight_attributes attributes) { + if (attributes.name.empty()) { + attributes.name += std::to_string(sub_nodes.size()); + } + // Make required output tensors + auto DBIAS = attributes.outputs[DBN_weight_attributes::output_names::DBIAS] = + output_tensor(attributes.name + "::DBIAS"); + auto DSCALE = attributes.outputs[DBN_weight_attributes::output_names::DSCALE] = + output_tensor(attributes.name + "::DSCALE"); + auto EQ_BIAS = attributes.outputs[DBN_weight_attributes::output_names::EQ_BIAS] = + output_tensor(attributes.name + "::EQ_BIAS"); + auto EQ_SCALE_DY = attributes.outputs[DBN_weight_attributes::output_names::EQ_SCALE_DY] = + output_tensor(attributes.name + "::EQ_SCALE_DY"); + auto EQ_SCALE_X = attributes.outputs[DBN_weight_attributes::output_names::EQ_SCALE_X] = + output_tensor(attributes.name + "::EQ_SCALE_X"); + + // Set inputs + attributes.inputs[DBN_weight_attributes::input_names::DY] = dy; + attributes.inputs[DBN_weight_attributes::input_names::X] = x; + attributes.inputs[DBN_weight_attributes::input_names::SCALE] = scale; + attributes.inputs[DBN_weight_attributes::input_names::MEAN] = mean; + attributes.inputs[DBN_weight_attributes::input_names::INV_VARIANCE] = inv_variance; + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + + return {DSCALE, DBIAS, EQ_SCALE_DY, EQ_SCALE_X, EQ_BIAS}; +} + +inline std::shared_ptr +Graph::conv_dgrad(std::shared_ptr dy, + std::shared_ptr w, + Conv_dgrad_attributes attributes) { + // Make required output tensors + auto DX = attributes.outputs[Conv_dgrad_attributes::output_names::DX] = output_tensor(attributes.name + "::DX"); + + // Set inputs + attributes.inputs[Conv_dgrad_attributes::input_names::DY] = dy; + attributes.inputs[Conv_dgrad_attributes::input_names::W] = w; + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + + return DX; +} + +inline std::array, 2> +Graph::genstats(std::shared_ptr x, Genstats_attributes attributes) { + // Set outputs + auto SUM = attributes.outputs[Genstats_attributes::output_names::SUM] = + output_tensor(attributes.name + "_sum_output"); + auto SQ_SUM = attributes.outputs[Genstats_attributes::output_names::SQ_SUM] = + output_tensor(attributes.name + "_sq_sum_output"); + + // Set inputs + attributes.inputs[Genstats_attributes::input_names::X] = x; + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + + return {SUM, SQ_SUM}; +} + +inline std::shared_ptr +Graph::conv_wgrad(std::shared_ptr dy, + std::shared_ptr x, + Conv_wgrad_attributes attributes) { + // Make required output tensors + auto DW = attributes.outputs[Conv_wgrad_attributes::output_names::DW] = output_tensor(attributes.name + "::DW"); + + // Set inputs + attributes.inputs[Conv_wgrad_attributes::input_names::X] = x; + attributes.inputs[Conv_wgrad_attributes::input_names::DY] = dy; + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + + return DW; +} + +inline std::array, 2> +Graph::rmsnorm(std::shared_ptr x, + std::shared_ptr scale, + Rmsnorm_attributes attributes) { + // Set outputs + auto Y = attributes.outputs[Rmsnorm_attributes::output_names::Y] = output_tensor(attributes.name + "::Y"); + std::shared_ptr INV_VARIANCE = nullptr; + if (attributes.forward_phase == NormFwdPhase_t::TRAINING) { + INV_VARIANCE = attributes.outputs[Rmsnorm_attributes::output_names::INV_VARIANCE] = + output_tensor(attributes.name + "::INV_VARIANCE"); + } + // Set inputs + attributes.inputs[Rmsnorm_attributes::input_names::X] = x; + attributes.inputs[Rmsnorm_attributes::input_names::SCALE] = scale; + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + + return {Y, INV_VARIANCE}; +} + +inline std::shared_ptr +Graph::rope(std::shared_ptr input, + std::shared_ptr freqs, + RoPE_attributes attributes) { + // RoPE writes to a user-bound buffer (no longer a virtual workspace tensor), + // because the rotated Q/K need to be saved across fwd→bwd for the bwd to consume them + // as inputs to SDPA bwd. + auto OUTPUT = attributes.outputs[RoPE_attributes::output_names::OUTPUT] = + output_tensor(attributes.name + "::OUTPUT"); + OUTPUT->set_is_virtual(false); + + // Set inputs + attributes.inputs[RoPE_attributes::input_names::INPUT] = input; + attributes.inputs[RoPE_attributes::input_names::FREQS] = freqs; + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + + return OUTPUT; +} + +inline std::shared_ptr +Graph::rope_backward(std::shared_ptr dy, + std::shared_ptr freqs, + RoPE_backward_attributes attributes) { + // dX is a real (user-bound) tensor by default — symmetric to fwd RoPE's Y output. + auto DX = attributes.outputs[RoPE_backward_attributes::output_names::DX] = output_tensor(attributes.name + "::DX"); + DX->set_is_virtual(false); + + attributes.inputs[RoPE_backward_attributes::input_names::DY] = dy; + attributes.inputs[RoPE_backward_attributes::input_names::FREQS] = freqs; + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + + return DX; +} + +inline std::array, 3> +Graph::rmsnorm_backward(std::shared_ptr dy, + std::shared_ptr x, + std::shared_ptr scale, + std::shared_ptr inv_variance, + Rmsnorm_backward_attributes attributes) { + // Set outputs + auto DX = attributes.outputs[Rmsnorm_backward_attributes::output_names::DX] = + output_tensor(attributes.name + "::DX"); + auto DScale = attributes.outputs[Rmsnorm_backward_attributes::output_names::DSCALE] = + output_tensor(attributes.name + "::Dscale"); + std::shared_ptr DBias = nullptr; + if (attributes.use_dbias.value_or(true)) { + DBias = attributes.outputs[Rmsnorm_backward_attributes::output_names::DBIAS] = + output_tensor(attributes.name + "::Dbias"); + } + + // Set inputs + attributes.inputs[Rmsnorm_backward_attributes::input_names::DY] = dy; + attributes.inputs[Rmsnorm_backward_attributes::input_names::X] = x; + attributes.inputs[Rmsnorm_backward_attributes::input_names::SCALE] = scale; + attributes.inputs[Rmsnorm_backward_attributes::input_names::INV_VARIANCE] = inv_variance; + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + + return {DX, DScale, DBias}; +} + +inline std::array, 2> +Graph::sdpa(std::shared_ptr q, + std::shared_ptr k, + std::shared_ptr v, + SDPA_attributes attributes) { + if (attributes.mma_core_mode == DataType_t::NOT_SET) { + attributes._set_mma_core_mode(DataType_t::HALF); + } + + // Call internal implementation and return only the O and Stats outputs for backward compatibility + auto internal_result = sdpa_internal(q, k, v, std::move(attributes)); + return {internal_result.O, internal_result.Stats}; +} + +inline std::array, 4> +Graph::sdpa_fp8(std::shared_ptr q, + std::shared_ptr k, + std::shared_ptr v, + std::shared_ptr descale_q, + std::shared_ptr descale_k, + std::shared_ptr descale_v, + std::shared_ptr descale_s, + std::shared_ptr scale_s, + std::shared_ptr scale_o, + SDPA_fp8_attributes attributes) { + if (attributes.mma_core_mode == DataType_t::NOT_SET) { + attributes._set_mma_core_mode(DataType_t::FP8_E4M3); + } + + // Set FP8 scaling inputs + attributes.inputs[SDPA_fp8_attributes::input_names::Descale_Q] = descale_q; + attributes.inputs[SDPA_fp8_attributes::input_names::Descale_K] = descale_k; + attributes.inputs[SDPA_fp8_attributes::input_names::Descale_V] = descale_v; + attributes.inputs[SDPA_fp8_attributes::input_names::Descale_S] = descale_s; + attributes.inputs[SDPA_fp8_attributes::input_names::Scale_S] = scale_s; + attributes.inputs[SDPA_fp8_attributes::input_names::Scale_O] = scale_o; + + // Call internal implementation and return {Output, Stats, Amax_S, Amax_O} as array for backward compatibility + auto internal_result = sdpa_internal(q, k, v, std::move(attributes)); + return {internal_result.O, internal_result.Stats, internal_result.Amax_S, internal_result.Amax_O}; +} + +inline std::array, 3> +Graph::sdpa_fp8(std::shared_ptr q, + std::shared_ptr k, + std::shared_ptr v, + std::shared_ptr descale_q, + std::shared_ptr descale_k, + std::shared_ptr descale_v, + SDPA_fp8_attributes attributes) { + if (attributes.mma_core_mode == DataType_t::NOT_SET) { + attributes._set_mma_core_mode(DataType_t::FP8_E4M3); + } + + // Set FP8 scaling inputs + attributes.inputs[SDPA_fp8_attributes::input_names::Descale_Q] = descale_q; + attributes.inputs[SDPA_fp8_attributes::input_names::Descale_K] = descale_k; + attributes.inputs[SDPA_fp8_attributes::input_names::Descale_V] = descale_v; + + // Call internal implementation and return {Output, Stats, Amax_O} as array for backward compatibility + auto internal_result = sdpa_internal(q, k, v, std::move(attributes)); + return {internal_result.O, internal_result.Stats, internal_result.Amax_O}; +} + +inline std::array, 7> +Graph::sdpa_fp8_backward(std::shared_ptr q, + std::shared_ptr k, + std::shared_ptr v, + std::shared_ptr o, + std::shared_ptr dO, + std::shared_ptr Stats, + std::shared_ptr descale_q, + std::shared_ptr descale_k, + std::shared_ptr descale_v, + std::shared_ptr descale_o, + std::shared_ptr descale_do, + std::shared_ptr descale_s, + std::shared_ptr descale_dp, + std::shared_ptr scale_s, + std::shared_ptr scale_dq, + std::shared_ptr scale_dk, + std::shared_ptr scale_dv, + std::shared_ptr scale_dp, + SDPA_fp8_backward_attributes attributes) { + // Make required output tensors + auto dQ = attributes.outputs[SDPA_fp8_backward_attributes::output_names::dQ] = + output_tensor(attributes.name + "::dQ"); + auto dK = attributes.outputs[SDPA_fp8_backward_attributes::output_names::dK] = + output_tensor(attributes.name + "::dK"); + auto dV = attributes.outputs[SDPA_fp8_backward_attributes::output_names::dV] = + output_tensor(attributes.name + "::dV"); + auto Amax_dQ = attributes.outputs[SDPA_fp8_backward_attributes::output_names::Amax_dQ] = + output_tensor(attributes.name + "::Amax_dQ"); + auto Amax_dK = attributes.outputs[SDPA_fp8_backward_attributes::output_names::Amax_dK] = + output_tensor(attributes.name + "::Amax_dK"); + auto Amax_dV = attributes.outputs[SDPA_fp8_backward_attributes::output_names::Amax_dV] = + output_tensor(attributes.name + "::Amax_dV"); + auto Amax_dP = attributes.outputs[SDPA_fp8_backward_attributes::output_names::Amax_dP] = + output_tensor(attributes.name + "::Amax_dP"); + + // Set inputs + attributes.inputs[SDPA_fp8_backward_attributes::input_names::Q] = q; + attributes.inputs[SDPA_fp8_backward_attributes::input_names::K] = k; + attributes.inputs[SDPA_fp8_backward_attributes::input_names::V] = v; + attributes.inputs[SDPA_fp8_backward_attributes::input_names::O] = o; + attributes.inputs[SDPA_fp8_backward_attributes::input_names::Stats] = Stats; + attributes.inputs[SDPA_fp8_backward_attributes::input_names::dO] = dO; + + attributes.inputs[SDPA_fp8_backward_attributes::input_names::Descale_Q] = descale_q; + attributes.inputs[SDPA_fp8_backward_attributes::input_names::Descale_K] = descale_k; + attributes.inputs[SDPA_fp8_backward_attributes::input_names::Descale_V] = descale_v; + attributes.inputs[SDPA_fp8_backward_attributes::input_names::Descale_S] = descale_s; + attributes.inputs[SDPA_fp8_backward_attributes::input_names::Descale_O] = descale_o; + attributes.inputs[SDPA_fp8_backward_attributes::input_names::Descale_dO] = descale_do; + attributes.inputs[SDPA_fp8_backward_attributes::input_names::Descale_dP] = descale_dp; + + attributes.inputs[SDPA_fp8_backward_attributes::input_names::Scale_dQ] = scale_dq; + attributes.inputs[SDPA_fp8_backward_attributes::input_names::Scale_dK] = scale_dk; + attributes.inputs[SDPA_fp8_backward_attributes::input_names::Scale_dV] = scale_dv; + attributes.inputs[SDPA_fp8_backward_attributes::input_names::Scale_S] = scale_s; + attributes.inputs[SDPA_fp8_backward_attributes::input_names::Scale_dP] = scale_dp; + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + + return {dQ, dK, dV, Amax_dQ, Amax_dK, Amax_dV, Amax_dP}; +} + +inline std::array, 6> +Graph::sdpa_fp8_backward(std::shared_ptr q, + std::shared_ptr q_T, + std::shared_ptr k, + std::shared_ptr k_T, + std::shared_ptr v, + std::shared_ptr o_f16, + std::shared_ptr dO_f16, + std::shared_ptr dO, + std::shared_ptr dO_T, + std::shared_ptr Stats, + std::shared_ptr descale_q, + std::shared_ptr descale_q_T, + std::shared_ptr descale_k, + std::shared_ptr descale_k_T, + std::shared_ptr descale_v, + std::shared_ptr descale_dO, + std::shared_ptr descale_dO_T, + SDPA_fp8_backward_attributes attributes) { + // Make required output tensors + auto dQ = attributes.outputs[SDPA_fp8_backward_attributes::output_names::dQ] = + output_tensor(attributes.name + "::dQ"); + auto dK = attributes.outputs[SDPA_fp8_backward_attributes::output_names::dK] = + output_tensor(attributes.name + "::dK"); + auto dV = attributes.outputs[SDPA_fp8_backward_attributes::output_names::dV] = + output_tensor(attributes.name + "::dV"); + auto Amax_dQ = attributes.outputs[SDPA_fp8_backward_attributes::output_names::Amax_dQ] = + output_tensor(attributes.name + "::Amax_dQ"); + auto Amax_dK = attributes.outputs[SDPA_fp8_backward_attributes::output_names::Amax_dK] = + output_tensor(attributes.name + "::Amax_dK"); + auto Amax_dV = attributes.outputs[SDPA_fp8_backward_attributes::output_names::Amax_dV] = + output_tensor(attributes.name + "::Amax_dV"); + + // Set inputs + attributes.inputs[SDPA_fp8_backward_attributes::input_names::Q] = q; + attributes.inputs[SDPA_fp8_backward_attributes::input_names::K] = k; + attributes.inputs[SDPA_fp8_backward_attributes::input_names::V] = v; + attributes.inputs[SDPA_fp8_backward_attributes::input_names::Q_T] = q_T; + attributes.inputs[SDPA_fp8_backward_attributes::input_names::K_T] = k_T; + attributes.inputs[SDPA_fp8_backward_attributes::input_names::O] = o_f16; + attributes.inputs[SDPA_fp8_backward_attributes::input_names::dO_f16] = dO_f16; + attributes.inputs[SDPA_fp8_backward_attributes::input_names::dO_T] = dO_T; + attributes.inputs[SDPA_fp8_backward_attributes::input_names::Stats] = Stats; + attributes.inputs[SDPA_fp8_backward_attributes::input_names::dO] = dO; + + attributes.inputs[SDPA_fp8_backward_attributes::input_names::Descale_Q] = descale_q; + attributes.inputs[SDPA_fp8_backward_attributes::input_names::Descale_Q_T] = descale_q_T; + attributes.inputs[SDPA_fp8_backward_attributes::input_names::Descale_K] = descale_k; + attributes.inputs[SDPA_fp8_backward_attributes::input_names::Descale_K_T] = descale_k_T; + attributes.inputs[SDPA_fp8_backward_attributes::input_names::Descale_V] = descale_v; + attributes.inputs[SDPA_fp8_backward_attributes::input_names::Descale_dO] = descale_dO; + attributes.inputs[SDPA_fp8_backward_attributes::input_names::Descale_dO_T] = descale_dO_T; + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + + return {dQ, dK, dV, Amax_dQ, Amax_dK, Amax_dV}; +} + +inline std::array, 3> +Graph::sdpa_backward(std::shared_ptr q, + std::shared_ptr k, + std::shared_ptr v, + std::shared_ptr o, + std::shared_ptr dO, + std::shared_ptr stats, + SDPA_backward_attributes attributes) { + // Set inputs + attributes.inputs[SDPA_backward_attributes::input_names::Q] = q; + attributes.inputs[SDPA_backward_attributes::input_names::K] = k; + attributes.inputs[SDPA_backward_attributes::input_names::V] = v; + attributes.inputs[SDPA_backward_attributes::input_names::O] = o; + attributes.inputs[SDPA_backward_attributes::input_names::dO] = dO; + attributes.inputs[SDPA_backward_attributes::input_names::Stats] = stats; + + // Make required output tensors + auto dQ = attributes.outputs[SDPA_backward_attributes::output_names::dQ] = output_tensor(attributes.name + "::dQ"); + auto dK = attributes.outputs[SDPA_backward_attributes::output_names::dK] = output_tensor(attributes.name + "::dK"); + auto dV = attributes.outputs[SDPA_backward_attributes::output_names::dV] = output_tensor(attributes.name + "::dV"); + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + + return {dQ, dK, dV}; +} + +inline std::shared_ptr +Graph::slice(std::shared_ptr input, Slice_attributes attributes) { + attributes.inputs[Slice_attributes::input_names::X] = input; + auto Y = attributes.outputs[Slice_attributes::output_names::Y] = output_tensor(attributes.name + "::Y"); + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + return Y; +} + +inline std::shared_ptr +Graph::transpose(std::shared_ptr input, Transpose_attributes attributes) { + attributes.inputs[Transpose_attributes::input_names::X] = input; + auto Y = attributes.outputs[Transpose_attributes::output_names::Y] = output_tensor(attributes.name + "::Y"); + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + return Y; +} + +inline std::array, 2> +Graph::block_scale_quantize(std::shared_ptr x, Block_scale_quantize_attributes attributes) { + // Set outputs + auto Y = attributes.outputs[Block_scale_quantize_attributes::output_names::Y] = + output_tensor(attributes.name + "::Y"); + auto scale = attributes.outputs[Block_scale_quantize_attributes::output_names::scale] = + output_tensor(attributes.name + "::scale"); + + // Set inputs + attributes.inputs[Block_scale_quantize_attributes::input_names::X] = x; + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + + return {Y, scale}; +} + +inline std::shared_ptr +Graph::block_scale_dequantize(std::shared_ptr x, + std::shared_ptr scale, + Block_scale_dequantize_attributes attributes) { + // Set outputs + auto Y = attributes.outputs[Block_scale_dequantize_attributes::output_names::Y] = + output_tensor(attributes.name + "::Y"); + + // Set inputs + attributes.inputs[Block_scale_dequantize_attributes::input_names::X] = x; + attributes.inputs[Block_scale_dequantize_attributes::input_names::scale] = scale; + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + + return Y; +} + +inline std::shared_ptr +Graph::concatenate(std::vector> x, Concatenate_attributes attributes) { + if (attributes.name.empty()) { + attributes.name += std::to_string(sub_nodes.size()); + } + + // Set outputs + auto Y = attributes.outputs[Concatenate_attributes::output_names::Y] = output_tensor(attributes.name + "::Y"); + + // Set inputs + for (auto &element : x) { + attributes.inputs.push_back(element); + } + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + + return Y; +} + +inline std::shared_ptr +Graph::moe_grouped_matmul(std::shared_ptr token, + std::shared_ptr weight, + std::shared_ptr first_token_offset, + std::shared_ptr token_index, + std::shared_ptr token_ks, + Moe_grouped_matmul_attributes attributes) { + if (attributes.name.empty()) { + attributes.name += std::to_string(sub_nodes.size()); + } + + auto output = attributes.outputs[Moe_grouped_matmul_attributes::output_names::Output] = + output_tensor(attributes.name + "::Output"); + + attributes.inputs[Moe_grouped_matmul_attributes::input_names::Token] = token; + attributes.inputs[Moe_grouped_matmul_attributes::input_names::Weight] = weight; + attributes.inputs[Moe_grouped_matmul_attributes::input_names::FirstTokenOffset] = first_token_offset; + attributes.inputs[Moe_grouped_matmul_attributes::input_names::TokenIndex] = token_index; + attributes.inputs[Moe_grouped_matmul_attributes::input_names::TokenKs] = token_ks; + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + + return output; +} + +inline std::shared_ptr +Graph::moe_grouped_matmul_bwd(std::shared_ptr doutput, + std::shared_ptr token, + std::shared_ptr first_token_offset, + Moe_grouped_matmul_bwd_attributes attributes) { + if (attributes.name.empty()) { + attributes.name += std::to_string(sub_nodes.size()); + } + + auto dweight = attributes.outputs[Moe_grouped_matmul_bwd_attributes::output_names::DWeight] = + output_tensor(attributes.name + "::DWeight"); + + attributes.inputs[Moe_grouped_matmul_bwd_attributes::input_names::DOutput] = doutput; + attributes.inputs[Moe_grouped_matmul_bwd_attributes::input_names::Token] = token; + attributes.inputs[Moe_grouped_matmul_bwd_attributes::input_names::FirstTokenOffset] = first_token_offset; + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + + return dweight; +} + +static inline std::ostream & +operator<<(std::ostream &os, Graph const &graph) { + os << graph.print(); + return os; +} + +} // namespace cudnn_frontend::graph diff --git a/deps/cudnn-frontend/include/cudnn_frontend/graph_properties.h b/deps/cudnn-frontend/include/cudnn_frontend/graph_properties.h new file mode 100644 index 0000000..c5dd2a4 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/graph_properties.h @@ -0,0 +1,3143 @@ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "context.h" + +#include "../cudnn_frontend_utils.h" + +namespace cudnn_frontend { + +namespace graph { + +inline std::optional +get_rescale_threshold_from_env() { + auto const* env_value = get_environment("CUDNN_RESCALE_THRESHOLD"); + if (env_value == nullptr || env_value[0] == '\0') { + return std::nullopt; + } + + errno = 0; + char* end = nullptr; + auto value = std::strtof(env_value, &end); + if (env_value == end || (end != nullptr && *end != '\0') || errno == ERANGE) { + return std::nullopt; + } + + return value; +} + +using managed_backend_descriptor_t = std::vector; + +// Enum to distinguish between runtime parameters and compile-time constants +enum class ScalarType { + RUNTIME_PARAM, // Value provided at execution time (can change) + COMPILE_TIME_CONST // Value baked into graph (fixed, optimizable) +}; + +// simple structure to hold all properties of a tensor. +// Each property has a getter setter. +class Tensor_attributes { + public: + using uid_t = int64_t; + + // There are two usecases of pass by value tensors: + // 1. Fused scalar constants + // 2. Scalar passed during execution + // In approach 1, users provide a value to embed into the graph. + // In approach 2, users set is_pass_by_value boolean and then pass a pointer to scalar value with execute() API. + // A closed set of types that are allowed to be passed by value. + using pass_by_values_t = std::variant; + + error_t + validate() const { + RETURN_CUDNN_FRONTEND_ERROR_IF( + dim.empty(), error_code_t::ATTRIBUTE_NOT_SET, "Tensor '" + name + "' dims not set."); + RETURN_CUDNN_FRONTEND_ERROR_IF( + stride.empty(), error_code_t::ATTRIBUTE_NOT_SET, "Tensor '" + name + "' strides not set."); + RETURN_CUDNN_FRONTEND_ERROR_IF(dim.size() != stride.size(), + error_code_t::ATTRIBUTE_NOT_SET, + "Tensor '" + name + "' does not equal dimensionality in dim and stride."); + RETURN_CUDNN_FRONTEND_ERROR_IF( + is_virtual && is_pass_by_value, + error_code_t::ATTRIBUTE_NOT_SET, + "Tensor '" + name + "' can't be both virutal and pass_by_value at the same time."); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + pass_by_value.has_value() & (!is_pass_by_value), + error_code_t::ATTRIBUTE_NOT_SET, + "Tensor '" + name + "' can't be a fused scalar and not a pass_by_value tensor at the same time."); + + // Validate compile-time constant constraints + RETURN_CUDNN_FRONTEND_ERROR_IF( + has_compile_time_constant && !is_pass_by_value, + error_code_t::ATTRIBUTE_NOT_SET, + "Tensor '" + name + "' with compile-time constant must have is_pass_by_value=true."); + + RETURN_CUDNN_FRONTEND_ERROR_IF(has_compile_time_constant && is_virtual, + error_code_t::ATTRIBUTE_NOT_SET, + "Tensor '" + name + "' can't be both compile-time constant and virtual."); + + // Can't have both compile-time constant and runtime parameter + RETURN_CUDNN_FRONTEND_ERROR_IF( + has_compile_time_constant && pass_by_value.has_value(), + error_code_t::ATTRIBUTE_NOT_SET, + "Tensor '" + name + "' can't have both compile-time constant and runtime pass_by_value."); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + has_ragged_offset_multiplier() && !ragged_offset, + error_code_t::ATTRIBUTE_NOT_SET, + "Tensor '" + name + "' with ragged offset multiplier must also have a ragged offset tensor."); + + RETURN_CUDNN_FRONTEND_ERROR_IF(ragged_offset_multiplier <= 0, + error_code_t::INVALID_VALUE, + "Tensor '" + name + "' ragged offset multiplier must be positive."); + + return {error_code_t::OK, ""}; + } + + private: + template + friend class Attributes; + + std::string name; + DataType_t data_type = DataType_t::NOT_SET; + std::vector dim = {}; + std::vector stride = {}; + bool is_virtual = false; + + std::optional pass_by_value = std::nullopt; + bool is_pass_by_value = false; + + // Compile-time constant support (distinct from pass_by_value, which is for runtime parameters) + bool has_compile_time_constant = false; + std::optional compile_time_constant_value = std::nullopt; + + TensorReordering_t reordering_type = TensorReordering_t::NONE; + uid_t uid = 0; + bool uid_assigned = false; + + std::shared_ptr ragged_offset; + int64_t ragged_offset_multiplier = 1; + int64_t alignment = 16; // Default to 16 bytes + int64_t vector_count = 1; // Default to 1 (no vectorization) + int64_t vector_dimension = -1; // Default to -1 (not set) + + auto + fill_from_context(detail::Context const& context) -> Tensor_attributes& { + if (get_data_type() == DataType_t::NOT_SET) { + if (get_is_virtual()) { + set_data_type(context.get_intermediate_data_type()); + } else { + set_data_type(context.get_io_data_type()); + } + } + return *this; + } + + public: + // Serialization functions +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + friend void + to_json(nlohmann::json& j, const Tensor_attributes& ta); + friend void + from_json(const nlohmann::json& j, Tensor_attributes& ta); +#endif + + Tensor_attributes() = default; + + Tensor_attributes(float const& scalar) { + pass_by_value = scalar; + is_pass_by_value = true; + dim = stride = {1}; + data_type = DataType_t::FLOAT; + } + + Tensor_attributes(half const& scalar) { + pass_by_value = scalar; + is_pass_by_value = true; + dim = stride = {1}; + data_type = DataType_t::HALF; + } + + Tensor_attributes(nv_bfloat16 const& scalar) { + pass_by_value = scalar; + is_pass_by_value = true; + dim = stride = {1}; + data_type = DataType_t::BFLOAT16; + } + + Tensor_attributes(int32_t const& scalar) { + pass_by_value = scalar; + is_pass_by_value = true; + dim = stride = {1}; + data_type = DataType_t::INT32; + } + + Tensor_attributes(int64_t const& scalar) { + pass_by_value = scalar; + is_pass_by_value = true; + dim = stride = {1}; + data_type = DataType_t::INT64; + } + + Tensor_attributes(double const& scalar) { + pass_by_value = scalar; + is_pass_by_value = true; + dim = stride = {1}; + data_type = DataType_t::DOUBLE; + } + + // Constructors with ScalarType for compile-time constant or runtime parameter control + Tensor_attributes(float const& scalar, ScalarType scalar_type) { + if (scalar_type == ScalarType::COMPILE_TIME_CONST) { + compile_time_constant_value = scalar; + has_compile_time_constant = true; + } else { + pass_by_value = scalar; + } + is_pass_by_value = true; + dim = stride = {1}; + data_type = DataType_t::FLOAT; + } + + Tensor_attributes(half const& scalar, ScalarType scalar_type) { + if (scalar_type == ScalarType::COMPILE_TIME_CONST) { + compile_time_constant_value = scalar; + has_compile_time_constant = true; + } else { + pass_by_value = scalar; + } + is_pass_by_value = true; + dim = stride = {1}; + data_type = DataType_t::HALF; + } + + Tensor_attributes(nv_bfloat16 const& scalar, ScalarType scalar_type) { + if (scalar_type == ScalarType::COMPILE_TIME_CONST) { + compile_time_constant_value = scalar; + has_compile_time_constant = true; + } else { + pass_by_value = scalar; + } + is_pass_by_value = true; + dim = stride = {1}; + data_type = DataType_t::BFLOAT16; + } + + Tensor_attributes(int32_t const& scalar, ScalarType scalar_type) { + if (scalar_type == ScalarType::COMPILE_TIME_CONST) { + compile_time_constant_value = scalar; + has_compile_time_constant = true; + } else { + pass_by_value = scalar; + } + is_pass_by_value = true; + dim = stride = {1}; + data_type = DataType_t::INT32; + } + + Tensor_attributes(int64_t const& scalar, ScalarType scalar_type) { + if (scalar_type == ScalarType::COMPILE_TIME_CONST) { + compile_time_constant_value = scalar; + has_compile_time_constant = true; + } else { + pass_by_value = scalar; + } + is_pass_by_value = true; + dim = stride = {1}; + data_type = DataType_t::INT64; + } + + Tensor_attributes(double const& scalar, ScalarType scalar_type) { + if (scalar_type == ScalarType::COMPILE_TIME_CONST) { + compile_time_constant_value = scalar; + has_compile_time_constant = true; + } else { + pass_by_value = scalar; + } + is_pass_by_value = true; + dim = stride = {1}; + data_type = DataType_t::DOUBLE; + } + + std::string + get_name() const { + return name; + } + + auto + set_name(std::string const& value) -> Tensor_attributes& { + name = value; + return *this; + } + + DataType_t + get_data_type() const { + return data_type; + } + + auto + set_data_type(DataType_t const value) -> Tensor_attributes& { + data_type = value; + return *this; + } + + std::vector + get_dim() const { + return dim; + } + + auto + set_dim(std::vector const& value) -> Tensor_attributes& { + dim = value; + return *this; + } + + int64_t + get_volume() const { + int64_t volume = 1ul; + for (int64_t d : dim) { + volume *= d; + } + return volume; + } + + std::vector + get_stride() const { + return stride; + } + + auto + set_stride(std::vector const& value) -> Tensor_attributes& { + stride = value; + return *this; + } + + bool + get_is_virtual() const { + return is_virtual; + } + + std::shared_ptr + get_ragged_offset() { + return ragged_offset; + } + + int64_t + get_ragged_offset_multiplier() const { + return ragged_offset_multiplier; + } + + bool + has_ragged_offset_multiplier() const { + return ragged_offset_multiplier != 1; + } + + auto + set_is_virtual(bool const value) -> Tensor_attributes& { + is_virtual = value; + return *this; + } + + auto + set_output(bool const value) -> Tensor_attributes& { + return set_is_virtual(!value); + } + + std::optional + get_pass_by_value() const { + return pass_by_value; + } + + bool + get_is_pass_by_value() const { + return is_pass_by_value; + } + + auto + set_is_pass_by_value(bool const value) -> Tensor_attributes& { + is_pass_by_value = value; + return *this; + } + + // Compile-time constant accessors + bool + get_has_compile_time_constant() const { + return has_compile_time_constant; + } + + std::optional + get_compile_time_constant() const { + return compile_time_constant_value; + } + + auto + set_compile_time_constant(pass_by_values_t const& value) -> Tensor_attributes& { + compile_time_constant_value = value; + has_compile_time_constant = true; + if (!is_pass_by_value) { + is_pass_by_value = true; + } + return *this; + } + + auto + set_as_runtime_parameter() -> Tensor_attributes& { + is_pass_by_value = true; + has_compile_time_constant = false; + compile_time_constant_value = std::nullopt; + return *this; + } + + TensorReordering_t + get_reordering_type() const { + return reordering_type; + } + + auto + set_reordering_type(TensorReordering_t const value) -> Tensor_attributes& { + reordering_type = value; + return *this; + } + + int64_t + get_alignment() const { + return alignment; + } + + auto + set_alignment(int64_t const value) -> Tensor_attributes& { + alignment = value; + return *this; + } + + int64_t + get_vector_count() const { + return vector_count; + } + + int64_t + get_vector_dimension() const { + return vector_dimension; + } + + auto + set_vector_count_and_dimension(int64_t const count, int64_t const dimension) -> Tensor_attributes& { + vector_count = count; + vector_dimension = dimension; + return *this; + } + + uid_t + get_uid() const { + return uid; + } + + uid_t + has_uid() const { + return uid_assigned; + } + + auto + clear_uid(void) -> Tensor_attributes& { + uid = 0; + uid_assigned = false; + return *this; + } + + auto + set_uid(uid_t value) -> Tensor_attributes& { + uid = value; + uid_assigned = true; + return *this; + } + + auto + set_ragged_offset(std::shared_ptr const& value) -> Tensor_attributes& { + ragged_offset = value; + return *this; + } + + auto + set_ragged_offset_multiplier(int64_t value) -> Tensor_attributes& { + ragged_offset_multiplier = value; + return *this; + } +}; + +class Batchnorm_attributes; +class Batchnorm_backward_attributes; +class Concatenate_attributes; + +template +class Attributes { + DerivedT& + self() { + return *static_cast(this); + } + DerivedT const& + self() const { + return *static_cast(this); + } + + protected: + std::vector + get_non_virtual_uids() const { + std::vector non_virtual_uids; + auto derived = static_cast(this); + + // Compile-time constants are excluded from the variant pack + auto should_be_in_variant_pack = [](std::shared_ptr const& tensor) { + return tensor && tensor->get_is_virtual() == false && !tensor->get_has_compile_time_constant(); + }; + + if constexpr (std::is_same_v) { + for (auto tensor : derived->inputs) { + if (should_be_in_variant_pack(tensor)) { + non_virtual_uids.push_back(tensor->get_uid()); + if (auto ragged_offset = tensor->get_ragged_offset()) { + non_virtual_uids.push_back(ragged_offset->get_uid()); + } + } + } + } else { + for (auto& [name, tensor] : derived->inputs) { + (void)name; + if (should_be_in_variant_pack(tensor)) { + non_virtual_uids.push_back(tensor->get_uid()); + if (auto ragged_offset = tensor->get_ragged_offset()) { + non_virtual_uids.push_back(ragged_offset->get_uid()); + } + } + } + } + + for (auto& [name, tensor] : derived->outputs) { + (void)name; + if (should_be_in_variant_pack(tensor)) { + non_virtual_uids.push_back(tensor->get_uid()); + if (auto ragged_offset = tensor->get_ragged_offset()) { + non_virtual_uids.push_back(ragged_offset->get_uid()); + } + } + } + + // Handle special case of BN where peer_stats is also an input + if constexpr (std::is_same_v || + std::is_same_v) { + for (auto& tensor : derived->peer_stats) { + if (should_be_in_variant_pack(tensor)) { + non_virtual_uids.push_back(tensor->get_uid()); + if (auto ragged_offset = tensor->get_ragged_offset()) { + non_virtual_uids.push_back(ragged_offset->get_uid()); + } + } + } + } + + return non_virtual_uids; + } + + public: + error_t + fill_pass_by_value(std::unordered_map& + tensor_to_pass_by_value) const { + auto derived = static_cast(this); + if constexpr (std::is_same_v) { + for (auto& tensor : derived->inputs) { + if (tensor && tensor->get_pass_by_value().has_value()) { + tensor_to_pass_by_value.emplace(tensor->get_uid(), tensor->get_pass_by_value().value()); + } + } + } else { + for (auto& [name, tensor] : derived->inputs) { + (void)name; + if (tensor && tensor->get_pass_by_value().has_value()) { + tensor_to_pass_by_value.emplace(tensor->get_uid(), tensor->get_pass_by_value().value()); + } + } + } + + return {error_code_t::OK, ""}; + } + + void + fill_from_context(detail::Context const& context) { + auto derived = static_cast(this); + + if constexpr (std::is_same_v) { + for (auto& tensor : derived->inputs) { + if (tensor) { + tensor->fill_from_context(context); + } + } + } else { + for (auto& [name, tensor] : derived->inputs) { + (void)name; + if (tensor) { + tensor->fill_from_context(context); + } + } + } + for (auto& [name, tensor] : derived->outputs) { + (void)name; + if (tensor) { + tensor->fill_from_context(context); + } + } + // Handle special case of BN where peer_stats is also an input + if constexpr (std::is_same_v || + std::is_same_v) { + for (auto& tensor : derived->peer_stats) { + if (tensor) { + tensor->fill_from_context(context); + } + } + } + + if (compute_data_type == DataType_t::NOT_SET) { + set_compute_data_type(context.get_compute_data_type()); + } + + // Infer shape and stride for fused scalars (runtime params and compile-time constants). + // Fused scalars expand to match the dimensionality of non-scalar tensors; if all are scalars, keep 1D. + auto is_fused_scalar = [](std::shared_ptr const& tensor) { + return tensor && (tensor->get_pass_by_value().has_value() || tensor->get_has_compile_time_constant()); + }; + + int64_t number_of_dims = 1; + if constexpr (std::is_same_v) { + for (auto tensor : derived->inputs) { + if (tensor && !is_fused_scalar(tensor)) { + number_of_dims = tensor->get_dim().size(); + break; + } + } + } else { + for (auto [name, tensor] : derived->inputs) { + (void)name; + if (tensor && !is_fused_scalar(tensor)) { + number_of_dims = tensor->get_dim().size(); + break; + } + } + } + + // If number of dims is still 1, try to see if user set output dims. + if (number_of_dims == 1) { + for (auto [name, tensor] : derived->outputs) { + (void)name; + if (tensor && !is_fused_scalar(tensor)) { + number_of_dims = tensor->get_dim().size(); + break; + } + } + } + + // Expand fused scalar dimensions to match the number of dims of non-scalar tensors + if constexpr (std::is_same_v) { + for (auto tensor : derived->inputs) { + if (is_fused_scalar(tensor)) { + tensor->set_dim(std::vector(number_of_dims, 1)); + tensor->set_stride(std::vector(number_of_dims, 1)); + } + } + } else { + for (auto [name, tensor] : derived->inputs) { + (void)name; + if (is_fused_scalar(tensor)) { + tensor->set_dim(std::vector(number_of_dims, 1)); + tensor->set_stride(std::vector(number_of_dims, 1)); + } + } + } + } + + std::string name; + DataType_t compute_data_type = DataType_t::NOT_SET; + + DerivedT& + set_name(std::string const& value) { + name = value; + return self(); + } + + DerivedT& + set_compute_data_type(DataType_t value) { + compute_data_type = value; + return self(); + } +}; + +class BN_finalize_attributes : public Attributes { + friend class Attributes; + friend class BatchNormFinalizeNode; + friend class Graph; + + public: + enum class input_names { + SUM, + SQ_SUM, + SCALE, + BIAS, + EPSILON, + ACCUM_COUNT, + PREV_RUNNING_MEAN, + PREV_RUNNING_VAR, + MOMENTUM + }; + std::unordered_map> inputs; + enum class output_names { EQ_SCALE, EQ_BIAS, MEAN, INV_VARIANCE, NEXT_RUNNING_MEAN, NEXT_RUNNING_VAR }; + + NLOHMANN_DEFINE_TYPE_INTRUSIVE(BN_finalize_attributes, name, compute_data_type, inputs, outputs) + std::unordered_map> outputs; + + BN_finalize_attributes& + set_previous_running_stats(std::shared_ptr& mean, + std::shared_ptr& variance, + std::shared_ptr& momentum) { + inputs[BN_finalize_attributes::input_names::PREV_RUNNING_MEAN] = mean; + inputs[BN_finalize_attributes::input_names::PREV_RUNNING_VAR] = variance; + inputs[BN_finalize_attributes::input_names::MOMENTUM] = momentum; + return *this; + } +}; + +class Genstats_attributes : public Attributes { + friend class Attributes; + friend class GenstatsNode; + friend class Graph; + + public: + enum class input_names { X }; + std::unordered_map> inputs; + + enum class output_names { SUM, SQ_SUM }; + std::unordered_map> outputs; + NLOHMANN_DEFINE_TYPE_INTRUSIVE(Genstats_attributes, name, compute_data_type, inputs, outputs) +}; + +class Conv_fprop_attributes : public Attributes { + friend class Attributes; + friend class ConvolutionNode; + friend class Graph; + + std::vector pre_padding; + std::vector post_padding; + std::vector stride; + std::vector dilation; + + ConvolutionMode_t math_mode = ConvolutionMode_t::CROSS_CORRELATION; + + public: + enum class input_names { X, W }; + std::unordered_map> inputs; + enum class output_names { Y }; + std::unordered_map> outputs; + NLOHMANN_DEFINE_TYPE_INTRUSIVE(Conv_fprop_attributes, + name, + compute_data_type, + inputs, + outputs, + pre_padding, + post_padding, + stride, + dilation, + math_mode) + + ConvolutionMode_t + get_convolution_mode() const { + return math_mode; + } + + std::vector + get_pre_padding() const { + return pre_padding; + } + + std::vector + get_post_padding() const { + return post_padding; + } + + Conv_fprop_attributes& + set_padding(std::vector value) { + pre_padding = value; + post_padding = value; + return *this; + } + + Conv_fprop_attributes& + set_pre_padding(std::vector value) { + pre_padding = value; + return *this; + } + + Conv_fprop_attributes& + set_post_padding(std::vector value) { + post_padding = value; + return *this; + } + + Conv_fprop_attributes& + set_convolution_mode(ConvolutionMode_t mode_) { + math_mode = mode_; + return *this; + } + + std::vector + get_stride() const { + return stride; + } + + Conv_fprop_attributes& + set_stride(std::vector value) { + stride = value; + return *this; + } + + std::vector + get_dilation() const { + return dilation; + } + + Conv_fprop_attributes& + set_dilation(std::vector value) { + dilation = value; + return *this; + } +}; + +class Batchnorm_backward_attributes : public Attributes { + friend class Attributes; + friend class DBNNode; + friend class Graph; + + public: + enum class input_names { DY, X, SCALE, MEAN, INV_VARIANCE }; + std::unordered_map> inputs; + // Only special case where one of the inputs is a vector. + std::vector> peer_stats; + enum class output_names { DX, DSCALE, DBIAS }; + NLOHMANN_DEFINE_TYPE_INTRUSIVE(Batchnorm_backward_attributes, name, compute_data_type, inputs, peer_stats, outputs) + std::unordered_map> outputs; + + Batchnorm_backward_attributes& + set_saved_mean_and_inv_variance(std::shared_ptr mean, + std::shared_ptr inv_variance) { + inputs[Batchnorm_backward_attributes::input_names::MEAN] = mean; + inputs[Batchnorm_backward_attributes::input_names::INV_VARIANCE] = inv_variance; + return *this; + } + + Batchnorm_backward_attributes& + set_peer_stats(std::vector> const& input_peer_stats) { + peer_stats = input_peer_stats; + return *this; + } +}; + +class DBN_weight_attributes : public Attributes { + friend class Attributes; + friend class DBNWeightNode; + friend class Graph; + + public: + enum class input_names { DY, X, SCALE, MEAN, INV_VARIANCE }; + std::unordered_map> inputs; + enum class output_names { DSCALE, DBIAS, EQ_BIAS, EQ_SCALE_DY, EQ_SCALE_X }; + NLOHMANN_DEFINE_TYPE_INTRUSIVE(DBN_weight_attributes, name, compute_data_type, inputs, outputs) + std::unordered_map> outputs; +}; + +class Conv_dgrad_attributes : public Attributes { + friend class Attributes; + friend class DgradNode; + friend class Graph; + + std::vector pre_padding; + std::vector post_padding; + std::vector stride; + std::vector dilation; + + ConvolutionMode_t math_mode = ConvolutionMode_t::CROSS_CORRELATION; + + public: + enum class input_names { DY, W }; + std::unordered_map> inputs; + enum class output_names { DX }; + std::unordered_map> outputs; + NLOHMANN_DEFINE_TYPE_INTRUSIVE(Conv_dgrad_attributes, + name, + compute_data_type, + inputs, + outputs, + pre_padding, + post_padding, + stride, + dilation, + math_mode) + + ConvolutionMode_t + get_convolution_mode() const { + return math_mode; + } + + std::vector + get_pre_padding() const { + return pre_padding; + } + + std::vector + get_post_padding() const { + return post_padding; + } + + Conv_dgrad_attributes& + set_padding(std::vector value) { + pre_padding = value; + post_padding = value; + return *this; + } + + Conv_dgrad_attributes& + set_pre_padding(std::vector value) { + pre_padding = value; + return *this; + } + + Conv_dgrad_attributes& + set_post_padding(std::vector value) { + post_padding = value; + return *this; + } + + std::vector + get_stride() const { + return stride; + } + + Conv_dgrad_attributes& + set_convolution_mode(ConvolutionMode_t mode_) { + math_mode = mode_; + ; + return *this; + } + + Conv_dgrad_attributes& + set_stride(std::vector value) { + stride = value; + return *this; + } + + std::vector + get_dilation() const { + return dilation; + } + + Conv_dgrad_attributes& + set_dilation(std::vector value) { + dilation = value; + return *this; + } +}; + +class Matmul_fp8_attributes : public Attributes { + friend class Attributes; + friend class MatmulFP8Node; + friend class INode; + + double padding_value = 0.0; + + public: + enum class input_names { Descale_A, Descale_B, A, B, M_override, N_override, K_override, Scale_C }; + std::unordered_map> inputs; + enum class output_names { C, Amax_C }; + std::unordered_map> outputs; + NLOHMANN_DEFINE_TYPE_INTRUSIVE(Matmul_fp8_attributes, name, compute_data_type, inputs, outputs) + + Matmul_fp8_attributes& + set_m_override(std::shared_ptr const& value) { + inputs[input_names::M_override] = value; + return *this; + } + + Matmul_fp8_attributes& + set_n_override(std::shared_ptr const& value) { + inputs[input_names::N_override] = value; + return *this; + } + + Matmul_fp8_attributes& + set_k_override(std::shared_ptr const& value) { + inputs[input_names::K_override] = value; + return *this; + } + + Matmul_fp8_attributes& + set_padding(double const padding_val) { + padding_value = padding_val; + return *this; + } + + double + get_padding() const { + return padding_value; + } +}; + +class Matmul_attributes : public Attributes { + friend class Attributes; + friend class MatmulNode; + friend class INode; + + double padding_value = 0.0; + + public: + enum class input_names { A, B, M_override, N_override, K_override }; + std::unordered_map> inputs; + enum class output_names { C }; + std::unordered_map> outputs; + NLOHMANN_DEFINE_TYPE_INTRUSIVE(Matmul_attributes, name, compute_data_type, inputs, outputs, padding_value) + + Matmul_attributes& + clone_fp8_attributes(Matmul_fp8_attributes const& attributes) { + auto m_override = attributes.inputs.find(Matmul_fp8_attributes::input_names::M_override); + if (m_override != attributes.inputs.end()) { + set_m_override(m_override->second); + } + auto n_override = attributes.inputs.find(Matmul_fp8_attributes::input_names::N_override); + if (n_override != attributes.inputs.end()) { + set_n_override(n_override->second); + } + auto k_override = attributes.inputs.find(Matmul_fp8_attributes::input_names::K_override); + if (k_override != attributes.inputs.end()) { + set_k_override(k_override->second); + } + + set_padding(attributes.get_padding()); + + return *this; + } + + Matmul_attributes& + set_m_override(std::shared_ptr const& value) { + inputs[input_names::M_override] = value; + return *this; + } + + Matmul_attributes& + set_n_override(std::shared_ptr const& value) { + inputs[input_names::N_override] = value; + return *this; + } + + Matmul_attributes& + set_k_override(std::shared_ptr const& value) { + inputs[input_names::K_override] = value; + return *this; + } + + Matmul_attributes& + set_padding(double const padding_val) { + padding_value = padding_val; + return *this; + } +}; + +class Pointwise_attributes : public Attributes { + friend class Attributes; + friend class PointwiseNode; + friend class CompositeSoftmaxNode; + friend class INode; + + PointwiseMode_t mode = PointwiseMode_t::NOT_SET; + + std::optional axis; + + std::optional relu_lower_clip; + std::optional relu_upper_clip; + std::optional relu_lower_clip_slope; + + std::optional swish_beta; + std::optional elu_alpha; + std::optional softplus_beta; + + public: + enum class input_names { IN_0, IN_1, IN_2 }; + std::unordered_map> inputs; + enum class output_names { OUT_0 }; + std::unordered_map> outputs; + NLOHMANN_DEFINE_TYPE_INTRUSIVE(Pointwise_attributes, + name, + compute_data_type, + inputs, + outputs, + mode, + axis, + relu_lower_clip, + relu_upper_clip, + relu_lower_clip_slope, + swish_beta, + elu_alpha, + softplus_beta) + + Pointwise_attributes& + set_mode(PointwiseMode_t const value) { + mode = value; + return *this; + } + + PointwiseMode_t + get_mode() const { + return mode; + } + + std::optional + get_axis() const { + return axis; + } + + Pointwise_attributes& + set_axis(int64_t const axis) { + this->axis = axis; + return *this; + } + + Pointwise_attributes& + set_relu_lower_clip_slope(float const negative_slope) { + this->relu_lower_clip_slope = negative_slope; + return *this; + } + + Pointwise_attributes& + set_relu_lower_clip(float const value) { + this->relu_lower_clip = value; + return *this; + } + + Pointwise_attributes& + set_relu_upper_clip(float const value) { + this->relu_upper_clip = value; + return *this; + } + + Pointwise_attributes& + set_swish_beta(float const value) { + this->swish_beta = value; + return *this; + } + + std::optional + get_swish_beta() const { + return swish_beta; + } + + Pointwise_attributes& + set_elu_alpha(float const value) { + this->elu_alpha = value; + return *this; + } + + Pointwise_attributes& + set_softplus_beta(float const value) { + this->softplus_beta = value; + return *this; + } +}; + +class Instancenorm_backward_attributes : public Attributes { + friend class Attributes; + friend class DINNode; + friend class Graph; + + public: + enum class input_names { DY, X, SCALE, MEAN, INV_VARIANCE }; + std::unordered_map> inputs; + enum class output_names { DX, DSCALE, DBIAS }; + std::unordered_map> outputs; + NLOHMANN_DEFINE_TYPE_INTRUSIVE(Instancenorm_backward_attributes, name, compute_data_type, inputs, outputs) + + Instancenorm_backward_attributes& + set_saved_mean_and_inv_variance(std::shared_ptr mean, + std::shared_ptr inv_variance) { + inputs[Instancenorm_backward_attributes::input_names::MEAN] = mean; + inputs[Instancenorm_backward_attributes::input_names::INV_VARIANCE] = inv_variance; + return *this; + } +}; + +class Layernorm_backward_attributes : public Attributes { + friend class Attributes; + friend class DLNNode; + friend class Graph; + + public: + enum class input_names { DY, X, SCALE, MEAN, INV_VARIANCE, EPSILON }; + std::unordered_map> inputs; + enum class output_names { DX, DSCALE, DBIAS }; + std::unordered_map> outputs; + NLOHMANN_DEFINE_TYPE_INTRUSIVE(Layernorm_backward_attributes, name, compute_data_type, inputs, outputs) + + Layernorm_backward_attributes& + set_saved_mean_and_inv_variance(std::shared_ptr mean, + std::shared_ptr inv_variance) { + inputs[Layernorm_backward_attributes::input_names::MEAN] = mean; + inputs[Layernorm_backward_attributes::input_names::INV_VARIANCE] = inv_variance; + return *this; + } +}; + +class Layernorm_attributes : public Attributes { + friend class Attributes; + friend class LayerNormNode; + friend class Graph; + + NormFwdPhase_t forward_phase = NormFwdPhase_t::NOT_SET; + + public: + enum class input_names { X, SCALE, BIAS, EPSILON }; + std::unordered_map> inputs; + enum class output_names { Y, MEAN, INV_VARIANCE }; + std::unordered_map> outputs; + NLOHMANN_DEFINE_TYPE_INTRUSIVE(Layernorm_attributes, name, compute_data_type, inputs, outputs, forward_phase) + + Layernorm_attributes& + set_forward_phase(NormFwdPhase_t const value) { + forward_phase = value; + return *this; + } + + Layernorm_attributes& + set_epsilon(std::shared_ptr& value) { + inputs[Layernorm_attributes::input_names::EPSILON] = value; + return *this; + } + + Layernorm_attributes& + set_epsilon(float const value) { + inputs[Layernorm_attributes::input_names::EPSILON] = std::make_shared(value); + return *this; + } +}; + +class AdaLayernorm_attributes : public Attributes { + friend class Attributes; + friend class AdaLayerNormNode; + friend class Graph; + + NormFwdPhase_t forward_phase = NormFwdPhase_t::NOT_SET; + + public: + enum class input_names { X, SCALE, BIAS, EPSILON }; + std::unordered_map> inputs; + enum class output_names { Y, MEAN, INV_VARIANCE }; + std::unordered_map> outputs; + NLOHMANN_DEFINE_TYPE_INTRUSIVE(AdaLayernorm_attributes, name, compute_data_type, inputs, outputs, forward_phase) + + AdaLayernorm_attributes& + set_forward_phase(NormFwdPhase_t const value) { + forward_phase = value; + return *this; + } + + AdaLayernorm_attributes& + set_epsilon(std::shared_ptr value) { + inputs[AdaLayernorm_attributes::input_names::EPSILON] = std::move(value); + return *this; + } +}; + +class AdaLayernorm_backward_attributes : public Attributes { + friend class Attributes; + friend class DAdaLayerNormNode; + friend class Graph; + + public: + enum class input_names { DY, X, SCALE, MEAN, INV_VARIANCE, EPSILON }; + std::unordered_map> inputs; + enum class output_names { DX, DSCALE, DBIAS }; + std::unordered_map> outputs; + NLOHMANN_DEFINE_TYPE_INTRUSIVE(AdaLayernorm_backward_attributes, name, compute_data_type, inputs, outputs) + + AdaLayernorm_backward_attributes& + set_saved_mean_and_inv_variance(std::shared_ptr mean, + std::shared_ptr inv_variance) { + inputs[AdaLayernorm_backward_attributes::input_names::MEAN] = mean; + inputs[AdaLayernorm_backward_attributes::input_names::INV_VARIANCE] = inv_variance; + return *this; + } +}; + +class Instancenorm_attributes : public Attributes { + friend class Attributes; + friend class InstanceNormNode; + friend class Graph; + + NormFwdPhase_t forward_phase = NormFwdPhase_t::NOT_SET; + + public: + enum class input_names { X, SCALE, BIAS, EPSILON }; + std::unordered_map> inputs; + enum class output_names { Y, MEAN, INV_VARIANCE }; + std::unordered_map> outputs; + NLOHMANN_DEFINE_TYPE_INTRUSIVE(Instancenorm_attributes, name, compute_data_type, inputs, outputs, forward_phase) + + Instancenorm_attributes& + set_forward_phase(NormFwdPhase_t const value) { + forward_phase = value; + return *this; + } + + Instancenorm_attributes& + set_epsilon(std::shared_ptr& value) { + inputs[Instancenorm_attributes::input_names::EPSILON] = value; + return *this; + } +}; + +class Batchnorm_attributes : public Attributes { + friend class Attributes; + friend class BatchNormNode; + friend class Graph; + + public: + enum class input_names { X, SCALE, BIAS, PREV_RUNNING_MEAN, PREV_RUNNING_VAR, EPSILON, MOMENTUM }; + std::unordered_map> inputs; + // Only special case where one of the inputs is a vector. + std::vector> peer_stats; + enum class output_names { Y, MEAN, INV_VARIANCE, NEXT_RUNNING_MEAN, NEXT_RUNNING_VAR }; + std::unordered_map> outputs; + NLOHMANN_DEFINE_TYPE_INTRUSIVE(Batchnorm_attributes, name, compute_data_type, inputs, peer_stats, outputs) + + Batchnorm_attributes& + set_previous_running_stats(std::shared_ptr& mean, + std::shared_ptr& variance, + std::shared_ptr& momentum) { + inputs[input_names::PREV_RUNNING_MEAN] = mean; + inputs[input_names::PREV_RUNNING_VAR] = variance; + inputs[input_names::MOMENTUM] = momentum; + return *this; + } + + Batchnorm_attributes& + set_epsilon(std::shared_ptr& value) { + inputs[input_names::EPSILON] = value; + return *this; + } + + Batchnorm_attributes& + set_peer_stats(std::vector> const& input_peer_stats) { + peer_stats = input_peer_stats; + return *this; + } +}; + +class Batchnorm_inference_attributes : public Attributes { + friend class Attributes; + friend class BatchnormInferenceNode; + friend class Graph; + + public: + enum class input_names { X, MEAN, INV_VARIANCE, SCALE, BIAS }; + std::unordered_map> inputs; + enum class output_names { Y }; + std::unordered_map> outputs; + NLOHMANN_DEFINE_TYPE_INTRUSIVE(Batchnorm_inference_attributes, name, compute_data_type, inputs, outputs) +}; + +class Reduction_attributes : public Attributes { + friend class Attributes; + friend class ReductionNode; + friend class INode; + + std::optional mode; + bool is_deterministic = false; + + public: + enum class input_names { X, Group_offset }; + std::unordered_map> inputs; + enum class output_names { Y }; + std::unordered_map> outputs; + NLOHMANN_DEFINE_TYPE_INTRUSIVE(Reduction_attributes, + name, + compute_data_type, + inputs, + outputs, + mode, + is_deterministic) + + std::optional + get_mode() const { + return mode; + } + + Reduction_attributes& + set_mode(ReductionMode_t value) { + mode = value; + return *this; + } + + bool + get_is_deterministic() const { + return is_deterministic; + } + + Reduction_attributes& + set_is_deterministic(bool value) { + is_deterministic = value; + return *this; + } + + Reduction_attributes& + set_group_offset(std::shared_ptr value) { + inputs[input_names::Group_offset] = std::move(value); + return *this; + } +}; + +class Rng_attributes : public Attributes { + friend class Attributes; + friend class RngNode; + friend class INode; + + RngDistribution_t distribution = RngDistribution_t::NOT_SET; + std::vector dim = {}; + std::vector stride = {}; + std::optional seed; + std::optional bernoulli_probability; + + public: + enum class input_names { Seed, Offset }; + std::unordered_map> inputs; + enum class output_names { Y }; + std::unordered_map> outputs; + NLOHMANN_DEFINE_TYPE_INTRUSIVE(Rng_attributes, + name, + inputs, + outputs, + distribution, + dim, + stride, + seed, + bernoulli_probability) + + std::vector + get_dim() const { + return dim; + } + + auto + set_dim(std::vector const& value) -> Rng_attributes& { + dim = value; + return *this; + } + + std::vector + get_stride() const { + return stride; + } + + auto + set_stride(std::vector const& value) -> Rng_attributes& { + stride = value; + return *this; + } + + RngDistribution_t + get_distribution() const { + return distribution; + } + + Rng_attributes& + set_distribution(RngDistribution_t value) { + distribution = value; + return *this; + } + + std::optional + get_seed() const { + return seed; + } + + Rng_attributes& + set_seed(std::optional value) { + seed = value; + return *this; + } + + std::optional + get_bernoulli_probability() const { + return bernoulli_probability; + } + + Rng_attributes& + set_bernoulli_probability(std::optional value) { + bernoulli_probability = value; + return *this; + } +}; + +class Resample_attributes : public Attributes { + friend class Attributes; + friend class ResampleNode; + friend class INode; + + std::optional generate_index; + ResampleMode_t resample_mode; + PaddingMode_t padding_mode; + std::vector pre_padding; + std::vector post_padding; + std::vector stride; + std::vector window; + + public: + enum class input_names { X }; + std::unordered_map> inputs; + + enum class output_names { Y, Index }; + std::unordered_map> outputs; + + NLOHMANN_DEFINE_TYPE_INTRUSIVE(Resample_attributes, + name, + inputs, + outputs, + generate_index, + resample_mode, + padding_mode, + pre_padding, + post_padding, + stride, + window) + + auto + set_resampling_mode(ResampleMode_t const& value) -> Resample_attributes& { + resample_mode = value; + return *this; + } + + auto + set_padding_mode(PaddingMode_t const& value) -> Resample_attributes& { + padding_mode = value; + return *this; + } + + auto + set_window(std::vector const& value) -> Resample_attributes& { + window.resize(value.size()); + for (auto i = 0u; i < value.size(); i++) { + window[i].numerator = value[i]; + window[i].denominator = 1; + } + return *this; + } + + auto + set_window(std::vector const& value) -> Resample_attributes& { + window = value; + return *this; + } + + auto + set_stride(std::vector const& value) -> Resample_attributes& { + stride.resize(value.size()); + for (auto i = 0u; i < value.size(); i++) { + stride[i].numerator = value[i]; + stride[i].denominator = 1; + } + return *this; + } + + auto + set_stride(std::vector const& value) -> Resample_attributes& { + stride = value; + return *this; + } + + auto + set_pre_padding(std::vector const& value) -> Resample_attributes& { + pre_padding.resize(value.size()); + for (auto i = 0u; i < value.size(); i++) { + pre_padding[i].numerator = value[i]; + pre_padding[i].denominator = 1; + } + return *this; + } + + auto + set_pre_padding(std::vector const& value) -> Resample_attributes& { + pre_padding = value; + return *this; + } + + auto + set_post_padding(std::vector const& value) -> Resample_attributes& { + post_padding.resize(value.size()); + for (auto i = 0u; i < value.size(); i++) { + post_padding[i].numerator = value[i]; + post_padding[i].denominator = 1; + } + return *this; + } + + auto + set_post_padding(std::vector const& value) -> Resample_attributes& { + post_padding = value; + return *this; + } + + auto + set_generate_index(bool const value) -> Resample_attributes& { + generate_index = value; + return *this; + } + + [[deprecated]] auto + set_is_inference(bool const value) -> Resample_attributes& { + return set_generate_index(!value); + } +}; + +class Reshape_attributes : public Attributes { + friend class Attributes; + friend class ReshapeNode; + friend class INode; + + std::vector dim = {}; + std::vector stride = {}; + ReshapeMode_t reshape_mode = ReshapeMode_t::VIEW_ONLY; + + public: + enum class input_names { X }; + std::unordered_map> inputs; + enum class output_names { Y }; + std::unordered_map> outputs; + NLOHMANN_DEFINE_TYPE_INTRUSIVE(Reshape_attributes, + name, + compute_data_type, + inputs, + outputs, + dim, + stride, + reshape_mode) + + std::vector + get_dim() const { + return dim; + } + + auto + set_dim(std::vector const& value) -> Reshape_attributes& { + dim = value; + return *this; + } + + std::vector + get_stride() const { + return stride; + } + + auto + set_stride(std::vector const& value) -> Reshape_attributes& { + stride = value; + return *this; + } + + ReshapeMode_t + get_reshape_mode() const { + return reshape_mode; + } + + auto + set_reshape_mode(ReshapeMode_t const& value) -> Reshape_attributes& { + reshape_mode = value; + return *this; + } +}; + +class Transpose_attributes : public Attributes { + friend class Attributes; + friend class TransposeNode; + friend class Graph; + + std::vector permutation; + + public: + std::string + get_name() const { + return name; + } + + auto + set_name(std::string const& value) -> Transpose_attributes& { + name = value; + return *this; + } + + enum class input_names { X }; + std::unordered_map> inputs; + enum class output_names { Y }; + std::unordered_map> outputs; + NLOHMANN_DEFINE_TYPE_INTRUSIVE(Transpose_attributes, name, compute_data_type, inputs, outputs, permutation) + + std::vector + get_permutation() const { + return permutation; + } + + auto + set_permutation(std::vector const& value) -> Transpose_attributes& { + permutation = value; + return *this; + } +}; + +class Rmsnorm_attributes : public Attributes { + friend class Attributes; + friend class RMSNormNode; + friend class Graph; + + NormFwdPhase_t forward_phase = NormFwdPhase_t::NOT_SET; + + public: + enum class input_names { X, SCALE, BIAS, EPSILON }; + std::unordered_map> inputs; + enum class output_names { Y, INV_VARIANCE }; + std::unordered_map> outputs; + NLOHMANN_DEFINE_TYPE_INTRUSIVE(Rmsnorm_attributes, name, compute_data_type, inputs, outputs, forward_phase) + + Rmsnorm_attributes& + set_forward_phase(NormFwdPhase_t const value) { + forward_phase = value; + return *this; + } + + Rmsnorm_attributes& + set_bias(std::shared_ptr& value) { + inputs[Rmsnorm_attributes::input_names::BIAS] = value; + return *this; + } + + Rmsnorm_attributes& + set_epsilon(std::shared_ptr& value) { + inputs[Rmsnorm_attributes::input_names::EPSILON] = value; + return *this; + } +}; + +class RoPE_attributes : public Attributes { + friend class Attributes; + friend class RoPENode; + friend class Graph; + + public: + enum class input_names { INPUT, FREQS }; + std::unordered_map> inputs; + enum class output_names { OUTPUT }; + std::unordered_map> outputs; + + // Host-side scalar multiplied into cos/sin. Default 1.0 (identity). + // Use this to fold YARN mscale and/or attn_scale into RoPE so SDPA can skip its scale multiply. + float output_scale = 1.0f; + + // Rotation width (last K dims). 0 = use input's head_dim (full rotation). + // When < head_dim, the kernel rotates the last rope_dim dims and scaled-pass-through + // copies the first (head_dim - rope_dim) dims. The freqs tensor's last dim must equal + // rope_dim (not head_dim). Used for DSv3-style MLA (rope_dim=64, head_dim=192). + int64_t rope_dim = 0; + + RoPE_attributes& + set_output_scale(float scale) { + output_scale = scale; + return *this; + } + + RoPE_attributes& + set_rope_dim(int64_t dim) { + rope_dim = dim; + return *this; + } + + NLOHMANN_DEFINE_TYPE_INTRUSIVE(RoPE_attributes, name, compute_data_type, inputs, outputs, output_scale, rope_dim) +}; + +class RoPE_backward_attributes : public Attributes { + friend class Attributes; + friend class RoPEBackwardNode; + friend class Graph; + + public: + enum class input_names { DY, FREQS }; + std::unordered_map> inputs; + enum class output_names { DX }; + std::unordered_map> outputs; + + // Same semantics as fwd: cos/sin pre-multiplied by output_scale, and the nope + // segment (head_dim - rope_dim) is scaled-pass-through. Should match the fwd's + // values for correct gradient flow. + float output_scale = 1.0f; + int64_t rope_dim = 0; + + RoPE_backward_attributes& + set_output_scale(float scale) { + output_scale = scale; + return *this; + } + + RoPE_backward_attributes& + set_rope_dim(int64_t dim) { + rope_dim = dim; + return *this; + } + + NLOHMANN_DEFINE_TYPE_INTRUSIVE(RoPE_backward_attributes, + name, + compute_data_type, + inputs, + outputs, + output_scale, + rope_dim) +}; + +class Rmsnorm_backward_attributes : public Attributes { + friend class Attributes; + friend class DRMSNormNode; + friend class Graph; + + std::optional use_dbias; + + public: + enum class input_names { DY, X, SCALE, INV_VARIANCE }; + std::unordered_map> inputs; + enum class output_names { DX, DSCALE, DBIAS }; + std::unordered_map> outputs; + NLOHMANN_DEFINE_TYPE_INTRUSIVE(Rmsnorm_backward_attributes, name, compute_data_type, inputs, outputs) + + Rmsnorm_backward_attributes& + has_dbias(bool value) { + use_dbias = value; + return *this; + } +}; + +// class Scaled_dot_product_attention_attributes : public Operation { +// public: +// struct Inputs { +// std::shared_ptr Q; +// std::shared_ptr K; +// std::shared_ptr Attn_scale; +// std::shared_ptr Bias; // Optional bias after bmm1 +// std::shared_ptr V; +// std::shared_ptr SEQ_LEN_Q; +// std::shared_ptr SEQ_LEN_KV; +// std::shared_ptr Mask; +// std::shared_ptr Dropout_mask; +// std::shared_ptr Dropout_scale; +// } inputs; + +// struct Outputs { +// std::shared_ptr O; +// std::shared_ptr +// S; // softmax output dumped when is_inference false. Users first need to check whether its nullptr. +// } outputs; + +// std::optional is_inference; +// bool padding_mask = false; +// bool causal_mask = false; +// std::optional dropout_probability; +// int64_t seed; +// float dropout_scale = 1.f; + +// public: +// Scaled_dot_product_attention_attributes() : Operation(Tag::Scaled_dot_product_attention), is_inference(false) {} + +// Scaled_dot_product_attention_attributes& +// set_is_inference(bool const value) { +// is_inference = value; +// return *this; +// } + +// Scaled_dot_product_attention_attributes& +// set_seq_len_q(std::shared_ptr value) { +// inputs.SEQ_LEN_Q = value; +// return *this; +// } + +// Scaled_dot_product_attention_attributes& +// set_seq_len_kv(std::shared_ptr value) { +// inputs.SEQ_LEN_KV = value; +// return *this; +// } + +// Scaled_dot_product_attention_attributes& +// set_padding_mask(bool const value) { +// padding_mask = value; +// return *this; +// } + +// Scaled_dot_product_attention_attributes& +// set_causal_mask(bool const value) { +// causal_mask = value; +// return *this; +// } + +// Scaled_dot_product_attention_attributes& +// set_attn_scale(std::shared_ptr value) { +// inputs.Attn_scale = value; +// return *this; +// } + +// Scaled_dot_product_attention_attributes& +// set_bias(std::shared_ptr bias) { +// inputs.Bias = bias; +// return *this; +// } + +// Scaled_dot_product_attention_attributes& +// set_dropout(float const probability, int64_t const seed_) { +// dropout_probability = probability; +// seed = seed_; +// return *this; +// } + +// Scaled_dot_product_attention_attributes& +// set_dropout(std::shared_ptr mask, std::shared_ptr scale) { +// inputs.Dropout_mask = mask; +// inputs.Dropout_scale = scale; +// return *this; +// } + +// Scaled_dot_product_attention_attributes& +// set_compute_data_type(DataType_t const value) { +// compute_data_type = value; +// return *this; +// } + +// Scaled_dot_product_attention_attributes& +// set_name(std::string const& value) { +// name = value; +// return *this; +// } + +// Scaled_dot_product_attention_attributes& +// fill_from_context(detail::Context const& context) { +// // Fill node's tensors +// inputs.Q->fill_from_context(context); +// inputs.K->fill_from_context(context); +// inputs.V->fill_from_context(context); +// inputs.SEQ_LEN_Q->fill_from_context(context); +// inputs.SEQ_LEN_KV->fill_from_context(context); +// outputs.O->fill_from_context(context); + +// // Fill this node +// if (get_compute_data_type() == DataType_t::NOT_SET) { +// set_compute_data_type(context.get_compute_data_type()); +// } +// return *this; +// } +// }; +template +class SDPANodeBase; +class CompositeSDPANode; +class UnifiedSDPANode; + +class SDPA_attributes : public Attributes { + friend class Attributes; + friend class SDPANodeBase; + friend class CompositeSDPANode; + friend class SDPANodeBase; + friend class UnifiedSDPANode; + friend class Graph; + + using Tensor_t = std::shared_ptr; + using Graph_t = std::shared_ptr; + + using AttentionScoreModifier_t = + std::function, std::shared_ptr)>; + + std::optional generate_stats; + bool alibi_mask = false; + bool padding_mask = false; + std::optional left_bound; + std::optional right_bound; + DiagonalAlignment_t diagonal_alignment = DiagonalAlignment_t::TOP_LEFT; + std::optional dropout_probability; + std::optional attn_scale_value; + std::optional max_seq_len_kv; + AttentionScoreModifier_t attention_score_modifier = nullptr; + DataType_t mma_core_mode = DataType_t::NOT_SET; + + AttentionImplementation_t implementation = AttentionImplementation_t::AUTO; + + bool unfuse_fma = false; // For SM100: use __fmul_rn + __fadd_rn instead of ffma2 in softmax + + bool + has_bias() const { + return inputs.find(input_names::Bias) != inputs.end() && inputs.at(input_names::Bias) != nullptr; + } + + bool + has_causal_like_masking() const { + return right_bound.has_value(); + } + + bool + has_causal_mask_bottom_right() const { + return right_bound.has_value() && diagonal_alignment == DiagonalAlignment_t::BOTTOM_RIGHT; + } + + bool + has_sink_token() const { + return inputs.find(input_names::SINK_TOKEN) != inputs.end() && inputs.at(input_names::SINK_TOKEN) != nullptr; + } + + public: + enum class input_names { + Q, + K, + V, + Attn_scale, + Bias, + SEQ_LEN_Q, + SEQ_LEN_KV, + CU_SEQ_LEN_Q, + CU_SEQ_LEN_KV, + Seed, + Offset, + Dropout_mask, + Dropout_scale, + Page_table_K, + Page_table_V, + Block_mask, + // FP8-specific scaling inputs + Descale_Q, + Descale_K, + Descale_V, + Descale_S, + Scale_S, + Scale_O, + SINK_TOKEN, + }; + std::unordered_map> inputs; + enum class output_names { O, Stats, Max, Sum_exp, RNG_DUMP, Amax_S, Amax_O }; + std::unordered_map> outputs; + // Convenience struct for named access to SDPA outputs + struct SDPA_outputs { + std::shared_ptr O; ///< Main attention output tensor + std::shared_ptr Stats; ///< Statistics/softmax output (when generate_stats=true) + std::shared_ptr Max; ///< Max output tensor + std::shared_ptr Sum_exp; ///< Sum_exp output tensor + std::shared_ptr RNG_DUMP; ///< Random number generator dump for dropout + ///< check why we don't return RNG_DUMP this way + std::shared_ptr Amax_S; ///< FP8 absolute maximum for attention scores + std::shared_ptr Amax_O; ///< FP8 absolute maximum for output tensor + }; + + NLOHMANN_DEFINE_TYPE_INTRUSIVE(SDPA_attributes, + name, + inputs, + outputs, + generate_stats, + alibi_mask, + padding_mask, + dropout_probability, + attn_scale_value, + max_seq_len_kv, + mma_core_mode, + left_bound, + right_bound, + diagonal_alignment, + implementation) + + SDPA_attributes& + set_generate_stats(bool const value) { + generate_stats = value; + return *this; + } + + SDPA_attributes& + set_logit_max(std::shared_ptr value) { + outputs[SDPA_attributes::output_names::Max] = std::move(value); + return *this; + } + + SDPA_attributes& + set_score_sum_exp(std::shared_ptr value) { + outputs[SDPA_attributes::output_names::Sum_exp] = std::move(value); + return *this; + } + + [[deprecated]] SDPA_attributes& + set_is_inference(bool const value) { + return set_generate_stats(!value); + } + + SDPA_attributes& + set_attn_scale(std::shared_ptr value) { + inputs[SDPA_attributes::input_names::Attn_scale] = std::move(value); + return *this; + } + + SDPA_attributes& + set_attn_scale(float const value) { + attn_scale_value = value; + return *this; + } + + SDPA_attributes& + set_bias(std::shared_ptr value) { + inputs[SDPA_attributes::input_names::Bias] = std::move(value); + return *this; + } + + SDPA_attributes& + set_block_mask(std::shared_ptr value) { + inputs[SDPA_attributes::input_names::Block_mask] = std::move(value); + return *this; + } + + SDPA_attributes& + set_alibi_mask(bool const value) { + alibi_mask = value; + return *this; + } + + SDPA_attributes& + set_padding_mask(bool const value) { + padding_mask = value; + return *this; + } + + // Internal function - do not use directly in application code + SDPA_attributes& + _set_mma_core_mode(DataType_t const value) { + mma_core_mode = value; + return *this; + } + + SDPA_attributes& + set_seq_len_q(std::shared_ptr value) { + inputs[SDPA_attributes::input_names::SEQ_LEN_Q] = std::move(value); + return *this; + } + + SDPA_attributes& + set_seq_len_kv(std::shared_ptr value) { + inputs[SDPA_attributes::input_names::SEQ_LEN_KV] = std::move(value); + return *this; + } + + SDPA_attributes& + set_cu_seq_len_q(std::shared_ptr value) { + inputs[SDPA_attributes::input_names::CU_SEQ_LEN_Q] = std::move(value); + return *this; + } + + SDPA_attributes& + set_cu_seq_len_kv(std::shared_ptr value) { + inputs[SDPA_attributes::input_names::CU_SEQ_LEN_KV] = std::move(value); + return *this; + } + + SDPA_attributes& + set_diagonal_alignment(DiagonalAlignment_t const alignment) { + diagonal_alignment = alignment; + return *this; + } + + // Sets the diagonal position to top left and + // calls set_diagonal_band_right_bound(0) if no right_bound was specified + // TODO: Deprecate + SDPA_attributes& + set_causal_mask(bool const value) { + if (value) { + set_diagonal_alignment(DiagonalAlignment_t::TOP_LEFT); + set_diagonal_band_right_bound(0); + } + return *this; + } + + // Sets the diagonal position to the bottom right (on a per-sequence basis) + // and calls set_diagonal_band_right_bound(0) if no right_bound was specified + // TODO: Deprecate + SDPA_attributes& + set_causal_mask_bottom_right(bool const value) { + if (value) { + set_diagonal_alignment(DiagonalAlignment_t::BOTTOM_RIGHT); + set_diagonal_band_right_bound(0); + } + return *this; + } + + SDPA_attributes& + set_score_mod(AttentionScoreModifier_t fn) { + attention_score_modifier = std::move(fn); + return *this; + } + + // calls set_diagonal_band_left_bound(value) + // TODO: Deprecate + SDPA_attributes& + set_sliding_window_length(int const value) { + return set_diagonal_band_left_bound(value); + } + + SDPA_attributes& + set_diagonal_band_left_bound(int const value) { + left_bound = value; + return *this; + } + + SDPA_attributes& + set_diagonal_band_right_bound(int const value) { + right_bound = value; + return *this; + } + + SDPA_attributes& + set_dropout(float const probability, + std::shared_ptr seed, + std::shared_ptr offset) { + dropout_probability = probability; + inputs[SDPA_attributes::input_names::Seed] = std::move(seed); + inputs[SDPA_attributes::input_names::Offset] = std::move(offset); + return *this; + } + + SDPA_attributes& + set_dropout(std::shared_ptr mask, std::shared_ptr scale) { + inputs[SDPA_attributes::input_names::Dropout_mask] = std::move(mask); + inputs[SDPA_attributes::input_names::Dropout_scale] = std::move(scale); + return *this; + } + + // For debugging purposes only. + SDPA_attributes& + set_rng_dump(std::shared_ptr value) { + outputs[SDPA_attributes::output_names::RNG_DUMP] = std::move(value); + return *this; + } + + SDPA_attributes& + set_paged_attention_k_table(std::shared_ptr value) { + inputs[SDPA_attributes::input_names::Page_table_K] = std::move(value); + return *this; + } + + SDPA_attributes& + set_paged_attention_v_table(std::shared_ptr value) { + inputs[SDPA_attributes::input_names::Page_table_V] = std::move(value); + return *this; + } + + SDPA_attributes& + set_paged_attention_max_seq_len_kv(int const value) { + max_seq_len_kv = value; + return *this; + } + + SDPA_attributes& + set_sink_token(std::shared_ptr value) { + inputs[SDPA_attributes::input_names::SINK_TOKEN] = std::move(value); + return *this; + } + + SDPA_attributes& + set_implementation(AttentionImplementation_t value) { + implementation = value; + return *this; + } + + SDPA_attributes& + set_unfuse_fma(bool value) { + unfuse_fma = value; + return *this; + } + + // Implementation is in sdpa_support_surface.h + error_t + validate_sdpa_support_surface(const detail::Context& context, int64_t s_kv, bool is_paged_k, bool is_paged_v) const; + + // Internal function - do not use directly in application code + void + _auto_select_implementation(const detail::Context& context) { + if (verify_sdpa_support_surface_for_implementation(context, AttentionImplementation_t::UNIFIED).is_good()) { + implementation = AttentionImplementation_t::UNIFIED; + CUDNN_FE_LOG_LABEL_ENDL("INFO: Auto-selected SDPA implementation UNIFIED"); + } else if (verify_sdpa_support_surface_for_implementation(context, AttentionImplementation_t::COMPOSITE) + .is_good()) { + implementation = AttentionImplementation_t::COMPOSITE; + CUDNN_FE_LOG_LABEL_ENDL("INFO: Auto-selected SDPA implementation COMPOSITE"); + } else { + // Leave `implementation` with its previous value (usually AUTO). + CUDNN_FE_LOG_LABEL_ENDL("ERROR: No suitable SDPA implementation for given SDPA_attributes"); + } + } + + private: + // Check whether implementation `impl` supports the requested features. `impl` must not be AUTO. + // (The `implementation` member variable is ignored.) + error_t + verify_sdpa_support_surface_for_implementation(const detail::Context& context, + AttentionImplementation_t impl) const; +}; + +// Type alias for backward compatibility - SDPA_fp8_attributes is now an alias to SDPA_attributes +// All FP8 functionality is unified in SDPA_attributes with the mma_core_mode field +using SDPA_fp8_attributes = SDPA_attributes; + +class SDPA_backward_attributes : public Attributes { + friend class Attributes; + friend class CompositeSDPABackwardNode; + friend class Graph; + using Tensor_t = std::shared_ptr; + using Graph_t = std::shared_ptr; + + using AttentionScoreModifier_t = + std::function, std::shared_ptr)>; + + bool alibi_mask = false; + bool padding_mask = false; + std::optional left_bound; + std::optional right_bound; + DiagonalAlignment_t diagonal_alignment = DiagonalAlignment_t::TOP_LEFT; + + std::optional dropout_probability; + std::optional attn_scale_value; + + std::optional max_total_seq_len_q; + std::optional max_total_seq_len_kv; + + bool is_deterministic_algorithm = false; + AttentionScoreModifier_t attention_score_modifier = nullptr; + AttentionScoreModifier_t attention_score_modifier_bprop = nullptr; + + bool + has_causal_like_masking() const { + return right_bound.has_value(); + } + + bool + has_causal_mask_bottom_right() const { + return right_bound.has_value() && diagonal_alignment == DiagonalAlignment_t::BOTTOM_RIGHT; + } + + public: + enum class input_names { + Q, + K, + V, + O, + dO, + Stats, + Attn_scale, + Bias, + SEQ_LEN_Q, + SEQ_LEN_KV, + Seed, + Offset, + Dropout_mask, + Dropout_scale, + Dropout_scale_inv, + SINK_TOKEN, + }; + std::unordered_map> inputs; + enum class output_names { dQ, dK, dV, dBias, RNG_DUMP, DSINK_TOKEN }; + std::unordered_map> outputs; + NLOHMANN_DEFINE_TYPE_INTRUSIVE(SDPA_backward_attributes, + name, + inputs, + outputs, + alibi_mask, + padding_mask, + dropout_probability, + attn_scale_value, + left_bound, + right_bound, + diagonal_alignment, + max_total_seq_len_q, + max_total_seq_len_kv, + is_deterministic_algorithm) + + SDPA_backward_attributes& + set_attn_scale(std::shared_ptr value) { + inputs[SDPA_backward_attributes::input_names::Attn_scale] = value; + return *this; + } + + SDPA_backward_attributes& + set_attn_scale(float const value) { + attn_scale_value = value; + return *this; + } + + SDPA_backward_attributes& + set_bias(std::shared_ptr value) { + inputs[SDPA_backward_attributes::input_names::Bias] = value; + return *this; + } + + SDPA_backward_attributes& + set_dbias(std::shared_ptr value) { + outputs[SDPA_backward_attributes::output_names::dBias] = value; + return *this; + } + + SDPA_backward_attributes& + set_alibi_mask(bool const value) { + alibi_mask = value; + return *this; + } + + SDPA_backward_attributes& + set_padding_mask(bool const value) { + padding_mask = value; + return *this; + } + + SDPA_backward_attributes& + set_score_mod(AttentionScoreModifier_t fn) { + attention_score_modifier = std::move(fn); + return *this; + } + + SDPA_backward_attributes& + set_score_mod_bprop(AttentionScoreModifier_t fn) { + attention_score_modifier_bprop = std::move(fn); + return *this; + } + + SDPA_backward_attributes& + set_seq_len_q(std::shared_ptr value) { + inputs[SDPA_backward_attributes::input_names::SEQ_LEN_Q] = value; + return *this; + } + + SDPA_backward_attributes& + set_seq_len_kv(std::shared_ptr value) { + inputs[SDPA_backward_attributes::input_names::SEQ_LEN_KV] = value; + return *this; + } + + SDPA_backward_attributes& + set_max_total_seq_len_q(int64_t const value) { + max_total_seq_len_q = value; + return *this; + } + + SDPA_backward_attributes& + set_max_total_seq_len_kv(int64_t const value) { + max_total_seq_len_kv = value; + return *this; + } + + SDPA_backward_attributes& + set_diagonal_alignment(DiagonalAlignment_t const alignment) { + diagonal_alignment = alignment; + return *this; + } + + // Sets the diagonal position to top left and + // calls set_diagonal_band_right_bound(0) if no right_bound was specified + // TODO: Deprecate + SDPA_backward_attributes& + set_causal_mask(bool const value) { + if (value) { + set_diagonal_alignment(DiagonalAlignment_t::TOP_LEFT); + set_diagonal_band_right_bound(0); + } + return *this; + } + + // Sets the diagonal position to the bottom right (on a per-sequence basis) + // and calls set_diagonal_band_right_bound(0) if no right_bound was specified + // TODO: Deprecate + SDPA_backward_attributes& + set_causal_mask_bottom_right(bool const value) { + if (value) { + set_diagonal_alignment(DiagonalAlignment_t::BOTTOM_RIGHT); + set_diagonal_band_right_bound(0); + } + return *this; + } + + // calls set_diagonal_band_left_bound(value) + // TODO: Deprecate + SDPA_backward_attributes& + set_sliding_window_length(int const value) { + return set_diagonal_band_left_bound(value); + } + + SDPA_backward_attributes& + set_diagonal_band_left_bound(int const value) { + left_bound = value; + return *this; + } + + SDPA_backward_attributes& + set_diagonal_band_right_bound(int const value) { + right_bound = value; + return *this; + } + + SDPA_backward_attributes& + set_dropout(float const probability, + std::shared_ptr seed, + std::shared_ptr offset) { + dropout_probability = probability; + inputs[SDPA_backward_attributes::input_names::Seed] = seed; + inputs[SDPA_backward_attributes::input_names::Offset] = offset; + return *this; + } + + SDPA_backward_attributes& + set_dropout(std::shared_ptr mask, + std::shared_ptr scale, + std::shared_ptr scale_inv) { + inputs[SDPA_backward_attributes::input_names::Dropout_mask] = mask; + inputs[SDPA_backward_attributes::input_names::Dropout_scale] = scale; + inputs[SDPA_backward_attributes::input_names::Dropout_scale_inv] = scale_inv; + return *this; + } + + // For debugging purposes only. + SDPA_backward_attributes& + set_rng_dump(std::shared_ptr value) { + outputs[SDPA_backward_attributes::output_names::RNG_DUMP] = value; + return *this; + } + + SDPA_backward_attributes& + set_deterministic_algorithm(bool const value) { + is_deterministic_algorithm = value; + return *this; + } + + SDPA_backward_attributes& + set_sink_token(std::shared_ptr value) { + inputs[SDPA_backward_attributes::input_names::SINK_TOKEN] = value; + return *this; + } + + SDPA_backward_attributes& + set_dsink_token(std::shared_ptr value) { + outputs[SDPA_backward_attributes::output_names::DSINK_TOKEN] = value; + return *this; + } +}; + +class SDPA_fp8_backward_attributes : public Attributes { + friend class Attributes; + friend class SDPAFP8BackwardNode; + friend class Graph; + + bool padding_mask = false; + bool is_deterministic_algorithm = false; + std::optional left_bound; + std::optional right_bound; + DiagonalAlignment_t diagonal_alignment = DiagonalAlignment_t::TOP_LEFT; + + std::optional dropout_probability; + std::optional attn_scale_value; + + public: + enum class input_names { + Q, + Q_T, + K, + K_T, + V, + O, + dO, + dO_T, + dO_f16, + Stats, + Attn_scale, + Bias, + SEQ_LEN_Q, + SEQ_LEN_KV, + Seed, + Offset, + Dropout_mask, + Dropout_scale, + Dropout_scale_inv, + + Descale_Q, + Descale_K, + Descale_V, + Descale_O, + Descale_dO, + Descale_dO_T, + Descale_K_T, + Descale_Q_T, + Descale_S, + Descale_dP, + Scale_dQ, + Scale_dK, + Scale_dV, + Scale_S, + Scale_dP, + SINK_TOKEN, + }; + std::unordered_map> inputs; + + enum class output_names { dQ, dK, dV, Amax_dQ, Amax_dK, Amax_dV, Amax_dP, DSINK_TOKEN }; + std::unordered_map> outputs; + + NLOHMANN_DEFINE_TYPE_INTRUSIVE(SDPA_fp8_backward_attributes, + name, + compute_data_type, + inputs, + outputs, + padding_mask, + dropout_probability, + left_bound, + right_bound, + diagonal_alignment, + attn_scale_value, + is_deterministic_algorithm) + + SDPA_fp8_backward_attributes& + set_attn_scale(std::shared_ptr value) { + inputs[SDPA_fp8_backward_attributes::input_names::Attn_scale] = value; + return *this; + } + + SDPA_fp8_backward_attributes& + set_attn_scale(float const value) { + attn_scale_value = value; + return *this; + } + + SDPA_fp8_backward_attributes& + set_bias(std::shared_ptr value) { + inputs[SDPA_fp8_backward_attributes::input_names::Bias] = value; + return *this; + } + + SDPA_fp8_backward_attributes& + set_padding_mask(bool const value) { + padding_mask = value; + return *this; + } + + SDPA_fp8_backward_attributes& + set_seq_len_q(std::shared_ptr value) { + inputs[SDPA_fp8_backward_attributes::input_names::SEQ_LEN_Q] = value; + return *this; + } + + SDPA_fp8_backward_attributes& + set_seq_len_kv(std::shared_ptr value) { + inputs[SDPA_fp8_backward_attributes::input_names::SEQ_LEN_KV] = value; + return *this; + } + + bool + has_causal_like_masking() const { + return right_bound.has_value(); + } + + bool + has_causal_mask_bottom_right() const { + return right_bound.has_value() && diagonal_alignment == DiagonalAlignment_t::BOTTOM_RIGHT; + } + + SDPA_fp8_backward_attributes& + set_causal_mask(bool const value) { + if (value) { + set_diagonal_alignment(DiagonalAlignment_t::TOP_LEFT); + set_diagonal_band_right_bound(0); + } + return *this; + } + + SDPA_fp8_backward_attributes& + set_causal_mask_bottom_right(bool const value) { + if (value) { + set_diagonal_alignment(DiagonalAlignment_t::BOTTOM_RIGHT); + set_diagonal_band_right_bound(0); + } + return *this; + } + + SDPA_fp8_backward_attributes& + set_diagonal_alignment(DiagonalAlignment_t const alignment) { + diagonal_alignment = alignment; + return *this; + } + + SDPA_fp8_backward_attributes& + set_diagonal_band_right_bound(int const value) { + right_bound = value; + return *this; + } + + SDPA_fp8_backward_attributes& + set_diagonal_band_left_bound(int const value) { + left_bound = value; + return *this; + } + + bool + has_sliding_window() const { + return left_bound.has_value(); + } + + SDPA_fp8_backward_attributes& + set_dropout(float const probability, + std::shared_ptr seed, + std::shared_ptr offset) { + dropout_probability = probability; + inputs[SDPA_fp8_backward_attributes::input_names::Seed] = seed; + inputs[SDPA_fp8_backward_attributes::input_names::Offset] = offset; + return *this; + } + + SDPA_fp8_backward_attributes& + set_dropout(std::shared_ptr mask, + std::shared_ptr scale, + std::shared_ptr scale_inv) { + inputs[SDPA_fp8_backward_attributes::input_names::Dropout_mask] = mask; + inputs[SDPA_fp8_backward_attributes::input_names::Dropout_scale] = scale; + inputs[SDPA_fp8_backward_attributes::input_names::Dropout_scale_inv] = scale_inv; + return *this; + } + + SDPA_fp8_backward_attributes& + set_deterministic_algorithm(bool const value) { + is_deterministic_algorithm = value; + return *this; + } + + SDPA_fp8_backward_attributes& + set_sink_token(std::shared_ptr value) { + inputs[SDPA_fp8_backward_attributes::input_names::SINK_TOKEN] = std::move(value); + return *this; + } + + SDPA_fp8_backward_attributes& + set_dsink_token(std::shared_ptr value) { + outputs[SDPA_fp8_backward_attributes::output_names::DSINK_TOKEN] = std::move(value); + return *this; + } + + bool + has_sink_token() const { + return inputs.find(input_names::SINK_TOKEN) != inputs.end() && inputs.at(input_names::SINK_TOKEN) != nullptr; + } +}; + +using Scaled_dot_product_flash_attention_attributes [[deprecated]] = SDPA_attributes; +using Scaled_dot_product_flash_attention_backward_attributes [[deprecated]] = SDPA_backward_attributes; + +class CompositeSoftmaxNode; +class UnifiedSoftmaxNode; + +class Softmax_attributes : public Attributes { + friend class Attributes; + friend class CompositeSoftmaxNode; + friend class UnifiedSoftmaxNode; + friend class INode; + + public: + enum class input_names { P, SINK }; + std::unordered_map> inputs; + enum class output_names { S, Stats, Max, Sum_exp }; + std::unordered_map> outputs; + NLOHMANN_DEFINE_TYPE_INTRUSIVE(Softmax_attributes, name, compute_data_type, inputs, outputs) + + Softmax_attributes& + set_sink(std::shared_ptr value) { + inputs[Softmax_attributes::input_names::SINK] = value; + return *this; + } +}; + +template +class DiagonalBandMaskNodeBase; +class CompositeDiagonalBandMaskNode; +class UnifiedDiagonalBandMaskNode; + +// Diagonal band mask attributes. +// A diagonal band mask is *either* a left-bound mask or a right-bound mask. +// LeftBound and ShiftRightBound cannot both be set at the same time. +// If neither LeftBound nor ShiftRightBound is set, a right-bound mask is assumed. +class DiagonalBandMask_attributes : public Attributes { + friend class Attributes; + friend class CompositeDiagonalBandMaskNode; + friend class UnifiedDiagonalBandMaskNode; + + PointwiseMode_t comparison_mode = PointwiseMode_t::CMP_GT; + + public: + enum class input_names { X, SEQ_LEN_Q, SEQ_LEN_KV, CU_SEQ_LEN_Q, CU_SEQ_LEN_KV, LeftBound, ShiftRightBound, B }; + std::unordered_map> inputs; + enum class output_names { Y }; + std::unordered_map> outputs; + NLOHMANN_DEFINE_TYPE_INTRUSIVE(DiagonalBandMask_attributes, + name, + compute_data_type, + inputs, + outputs, + comparison_mode) + + PointwiseMode_t + get_comparison_mode() const { + return comparison_mode; + } + + DiagonalBandMask_attributes& + set_comparison_mode(PointwiseMode_t value) { + comparison_mode = value; + return *this; + } +}; + +class Conv_wgrad_attributes : public Attributes { + friend class Attributes; + friend class WgradNode; + friend class Graph; + + std::vector pre_padding; + std::vector post_padding; + std::vector stride; + std::vector dilation; + ConvolutionMode_t math_mode = ConvolutionMode_t::CROSS_CORRELATION; + + public: + enum class input_names { DY, X }; + std::unordered_map> inputs; + + enum class output_names { DW }; + std::unordered_map> outputs; + NLOHMANN_DEFINE_TYPE_INTRUSIVE(Conv_wgrad_attributes, + name, + compute_data_type, + inputs, + outputs, + pre_padding, + post_padding, + stride, + dilation, + math_mode) + + ConvolutionMode_t + get_convolution_mode() const { + return math_mode; + } + + std::vector + get_pre_padding() const { + return pre_padding; + } + + std::vector + get_post_padding() const { + return post_padding; + } + + Conv_wgrad_attributes& + set_convolution_mode(ConvolutionMode_t mode_) { + math_mode = mode_; + ; + return *this; + } + + Conv_wgrad_attributes& + set_padding(std::vector value) { + pre_padding = value; + post_padding = value; + return *this; + } + + Conv_wgrad_attributes& + set_pre_padding(std::vector value) { + pre_padding = value; + return *this; + } + + Conv_wgrad_attributes& + set_post_padding(std::vector value) { + post_padding = value; + return *this; + } + + std::vector + get_stride() const { + return stride; + } + + Conv_wgrad_attributes& + set_stride(std::vector value) { + stride = value; + return *this; + } + + std::vector + get_dilation() const { + return dilation; + } + + Conv_wgrad_attributes& + set_dilation(std::vector value) { + dilation = value; + return *this; + } +}; + +class Slice_attributes : public Attributes { + friend class Attributes; + friend class SliceNode; + friend class INode; + + std::vector> slices; + std::vector slice_strides = {1}; + + public: + enum class input_names { X }; + std::unordered_map> inputs; + enum class output_names { Y }; + std::unordered_map> outputs; + NLOHMANN_DEFINE_TYPE_INTRUSIVE(Slice_attributes, name, compute_data_type, inputs, outputs, slices, slice_strides) + + Slice_attributes& + set_slices(std::vector> const value) { + slices = value; + return *this; + } + + Slice_attributes& + set_strides(std::vector const value) { + slice_strides = value; + return *this; + } + + int64_t + get_offset() const { + auto& input = inputs.at(input_names::X); + auto const input_stride = input->get_stride(); + + int64_t offset = 0; + + // Get number of elements to skip + for (size_t i = 0; i < slices.size(); ++i) { + offset += slices[i].first * input_stride[i]; + } + + // multiply by element size to get offset in bytes + offset *= detail::get_data_type_size(input->get_data_type()); + return offset; + } +}; + +class PagedCacheLoad_attributes : public Attributes { + friend class Attributes; + friend class PagedCacheLoadNode; + friend class INode; + + public: + enum class input_names { container, seqLen, pageTable }; + std::unordered_map> inputs; + enum class output_names { yOut }; + std::unordered_map> outputs; + NLOHMANN_DEFINE_TYPE_INTRUSIVE(PagedCacheLoad_attributes, name, compute_data_type, inputs, outputs) +}; + +class Block_scale_quantize_attributes : public Attributes { + friend class Attributes; + friend class BlockScaleQuantizeNode; + friend class Graph; + + std::optional block_size; + std::optional axis; + bool transpose = false; + + public: + enum class input_names { X }; + std::unordered_map> inputs; + enum class output_names { Y, scale }; + std::unordered_map> outputs; + NLOHMANN_DEFINE_TYPE_INTRUSIVE(Block_scale_quantize_attributes, + name, + compute_data_type, + inputs, + outputs, + block_size, + axis) + + Block_scale_quantize_attributes& + set_block_size(int32_t const value) { + block_size = value; + return *this; + } + + Block_scale_quantize_attributes& + set_axis(int64_t const value) { + axis = value; + return *this; + } + + Block_scale_quantize_attributes& + set_transpose(bool const value) { + transpose = value; + return *this; + } +}; + +class Block_scale_dequantize_attributes : public Attributes { + friend class Attributes; + friend class BlockScaleDequantizeNode; + friend class Graph; + + std::vector block_size; + bool is_negative_scale; + + public: + enum class input_names { X, scale }; + std::unordered_map> inputs; + enum class output_names { Y }; + std::unordered_map> outputs; + NLOHMANN_DEFINE_TYPE_INTRUSIVE(Block_scale_dequantize_attributes, + name, + compute_data_type, + inputs, + outputs, + block_size, + is_negative_scale) + + Block_scale_dequantize_attributes& + set_block_size(int32_t const value, int32_t idx = 0) { + if (idx < 0) { + return *this; + } + if (static_cast(block_size.size()) < idx + 1) { + block_size.resize(idx + 1, 1); + } + block_size[idx] = value; + return *this; + } + + Block_scale_dequantize_attributes& + set_block_size(const int32_t* values, int32_t len = 1) { + if (len < 1) { + return *this; + } + if (static_cast(block_size.size()) < len) { + block_size.resize(len); + } + std::copy(values, values + len, block_size.begin()); + return *this; + } + + Block_scale_dequantize_attributes& + set_block_size(const std::vector& values) { + block_size = values; + return *this; + } + + bool + get_is_negative_scale() const { + return is_negative_scale; + } + + Block_scale_dequantize_attributes& + set_is_negative_scale(bool value) { + is_negative_scale = value; + return *this; + } +}; + +#if 0 +class Concatenate_string { + friend class Attributes; + friend class ConcatenateNode; + friend class Graph; +public: +std::string str; +NLOHMANN_DEFINE_TYPE_INTRUSIVE(Concatenate_string, str) +}; +#endif + +class Concatenate_attributes : public Attributes { + friend class Attributes; + friend class ConcatenateNode; + friend class Graph; + + std::optional axis; + std::optional in_place_index; + + public: + std::vector> inputs; + enum class output_names { Y }; + std::unordered_map> outputs; + NLOHMANN_DEFINE_TYPE_INTRUSIVE(Concatenate_attributes, name, inputs, outputs, axis, in_place_index) + + Concatenate_attributes& + set_axis(int64_t const value) { + axis = value; + return *this; + } + + Concatenate_attributes& + set_in_place_index(int64_t const value) { + in_place_index = value; + return *this; + } +}; + +class Moe_grouped_matmul_attributes : public Attributes { + friend class Attributes; + friend class MoeGroupedMatmulNode; + friend class Graph; + + MoeGroupedMatmulMode_t mode = MoeGroupedMatmulMode_t::NONE; + + int32_t top_k = 0; + + public: + enum class input_names { Token, Weight, FirstTokenOffset, TokenIndex, TokenKs }; + std::unordered_map> inputs; + enum class output_names { Output }; + std::unordered_map> outputs; + NLOHMANN_DEFINE_TYPE_INTRUSIVE(Moe_grouped_matmul_attributes, name, inputs, outputs, mode, top_k) + + Moe_grouped_matmul_attributes& + set_mode(MoeGroupedMatmulMode_t mode) { + this->mode = mode; + return *this; + } + + Moe_grouped_matmul_attributes& + set_top_k(int32_t top_k) { + this->top_k = top_k; + return *this; + } +}; + +class Moe_grouped_matmul_bwd_attributes : public Attributes { + friend class Attributes; + friend class MoeGroupedMatmulBwdNode; + friend class Graph; + + public: + enum class input_names { DOutput, Token, FirstTokenOffset }; + std::unordered_map> inputs; + enum class output_names { DWeight }; + std::unordered_map> outputs; + NLOHMANN_DEFINE_TYPE_INTRUSIVE(Moe_grouped_matmul_bwd_attributes, name, inputs, outputs) +}; + +} // namespace graph + +} // namespace cudnn_frontend diff --git a/deps/cudnn-frontend/include/cudnn_frontend/knobs.h b/deps/cudnn-frontend/include/cudnn_frontend/knobs.h new file mode 100644 index 0000000..559643a --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/knobs.h @@ -0,0 +1,254 @@ +#pragma once + +namespace cudnn_frontend { + +enum class KnobType_t { + NOT_SET, + + SWIZZLE, + TILE_SIZE, + EDGE, + MULTIPLY, + SPLIT_K_BUF, + TILEK, + STAGES, + REDUCTION_MODE, + SPLIT_K_SLC, + IDX_MODE, + SPECFILT, + KERNEL_CFG, + WORKSPACE, + TILE_CGA_M, + TILE_CGA_N, + BLOCK_SIZE, + OCCUPANCY, + ARRAY_SIZE_PER_THREAD, + SPLIT_COLS, + TILE_ROWS, + TILE_COLS, + LOAD_SIZE, + CTA_COUNT, + STREAM_K, + SPLIT_P_SLC, + TILE_M, + TILE_N, + WARP_SPEC_CFG, + SWAP_AB, + INPUT_TMA_ENABLE, + OUTPUT_TMA_ENABLE, +}; + +class Knob { + public: + KnobType_t type = KnobType_t::NOT_SET; + int64_t maxValue = 0; + int64_t minValue = 0; + int64_t stride = 0; + + Knob(KnobType_t type, int64_t max, int64_t min, int64_t str) + : type(type), maxValue(max), minValue(min), stride(str) {} +}; + +static inline cudnnStatus_t +convert_to_backend_knob_type(KnobType_t const knob_type, cudnnBackendKnobType_t& cudnn_knob_type) { + switch (knob_type) { + case KnobType_t::SWIZZLE: + cudnn_knob_type = CUDNN_KNOB_TYPE_SWIZZLE; + return cudnnStatus_t::CUDNN_STATUS_SUCCESS; + case KnobType_t::TILE_SIZE: + cudnn_knob_type = CUDNN_KNOB_TYPE_TILE_SIZE; + return cudnnStatus_t::CUDNN_STATUS_SUCCESS; + case KnobType_t::EDGE: + cudnn_knob_type = CUDNN_KNOB_TYPE_EDGE; + return cudnnStatus_t::CUDNN_STATUS_SUCCESS; + case KnobType_t::MULTIPLY: + cudnn_knob_type = CUDNN_KNOB_TYPE_MULTIPLY; + return cudnnStatus_t::CUDNN_STATUS_SUCCESS; + case KnobType_t::SPLIT_K_BUF: + cudnn_knob_type = CUDNN_KNOB_TYPE_SPLIT_K_BUF; + return cudnnStatus_t::CUDNN_STATUS_SUCCESS; + case KnobType_t::TILEK: + cudnn_knob_type = CUDNN_KNOB_TYPE_TILEK; + return cudnnStatus_t::CUDNN_STATUS_SUCCESS; + case KnobType_t::STAGES: + cudnn_knob_type = CUDNN_KNOB_TYPE_STAGES; + return cudnnStatus_t::CUDNN_STATUS_SUCCESS; + case KnobType_t::REDUCTION_MODE: + cudnn_knob_type = CUDNN_KNOB_TYPE_REDUCTION_MODE; + return cudnnStatus_t::CUDNN_STATUS_SUCCESS; + case KnobType_t::SPLIT_K_SLC: + cudnn_knob_type = CUDNN_KNOB_TYPE_SPLIT_K_SLC; + return cudnnStatus_t::CUDNN_STATUS_SUCCESS; + case KnobType_t::IDX_MODE: + cudnn_knob_type = CUDNN_KNOB_TYPE_IDX_MODE; + return cudnnStatus_t::CUDNN_STATUS_SUCCESS; + case KnobType_t::SPECFILT: + cudnn_knob_type = CUDNN_KNOB_TYPE_SPECFILT; + return cudnnStatus_t::CUDNN_STATUS_SUCCESS; + case KnobType_t::KERNEL_CFG: + cudnn_knob_type = CUDNN_KNOB_TYPE_KERNEL_CFG; + return cudnnStatus_t::CUDNN_STATUS_SUCCESS; + case KnobType_t::WORKSPACE: + cudnn_knob_type = CUDNN_KNOB_TYPE_WORKSPACE; + return cudnnStatus_t::CUDNN_STATUS_SUCCESS; +#if (CUDNN_VERSION >= 8600) + case KnobType_t::TILE_CGA_M: + cudnn_knob_type = CUDNN_KNOB_TYPE_TILE_CGA_M; + return cudnnStatus_t::CUDNN_STATUS_SUCCESS; + case KnobType_t::TILE_CGA_N: + cudnn_knob_type = CUDNN_KNOB_TYPE_TILE_CGA_N; + return cudnnStatus_t::CUDNN_STATUS_SUCCESS; +#endif +#if (CUDNN_VERSION >= 8800) + case KnobType_t::BLOCK_SIZE: + cudnn_knob_type = CUDNN_KNOB_TYPE_BLOCK_SIZE; + return cudnnStatus_t::CUDNN_STATUS_SUCCESS; +#endif +#if (CUDNN_VERSION >= 8900) + case KnobType_t::OCCUPANCY: + cudnn_knob_type = CUDNN_KNOB_TYPE_OCCUPANCY; + return cudnnStatus_t::CUDNN_STATUS_SUCCESS; + case KnobType_t::ARRAY_SIZE_PER_THREAD: + cudnn_knob_type = CUDNN_KNOB_TYPE_ARRAY_SIZE_PER_THREAD; + return cudnnStatus_t::CUDNN_STATUS_SUCCESS; +#endif +#if (CUDNN_VERSION >= 8905) + case KnobType_t::SPLIT_COLS: + cudnn_knob_type = CUDNN_KNOB_TYPE_SPLIT_COLS; + return cudnnStatus_t::CUDNN_STATUS_SUCCESS; + case KnobType_t::TILE_ROWS: + cudnn_knob_type = CUDNN_KNOB_TYPE_TILE_ROWS; + return cudnnStatus_t::CUDNN_STATUS_SUCCESS; + case KnobType_t::TILE_COLS: + cudnn_knob_type = CUDNN_KNOB_TYPE_TILE_COLS; + return cudnnStatus_t::CUDNN_STATUS_SUCCESS; + case KnobType_t::LOAD_SIZE: + cudnn_knob_type = CUDNN_KNOB_TYPE_LOAD_SIZE; + return cudnnStatus_t::CUDNN_STATUS_SUCCESS; +#endif +#if (CUDNN_VERSION >= 90700) + case KnobType_t::CTA_COUNT: + cudnn_knob_type = CUDNN_KNOB_TYPE_CTA_COUNT; + return cudnnStatus_t::CUDNN_STATUS_SUCCESS; + case KnobType_t::STREAM_K: + cudnn_knob_type = CUDNN_KNOB_TYPE_STREAM_K; + return cudnnStatus_t::CUDNN_STATUS_SUCCESS; + case KnobType_t::SPLIT_P_SLC: + cudnn_knob_type = CUDNN_KNOB_TYPE_SPLIT_P_SLC; + return cudnnStatus_t::CUDNN_STATUS_SUCCESS; + case KnobType_t::TILE_M: + cudnn_knob_type = CUDNN_KNOB_TYPE_TILE_M; + return cudnnStatus_t::CUDNN_STATUS_SUCCESS; + case KnobType_t::TILE_N: + cudnn_knob_type = CUDNN_KNOB_TYPE_TILE_N; + return cudnnStatus_t::CUDNN_STATUS_SUCCESS; + case KnobType_t::WARP_SPEC_CFG: + cudnn_knob_type = CUDNN_KNOB_TYPE_WARP_SPEC_CFG; + return cudnnStatus_t::CUDNN_STATUS_SUCCESS; +#endif +#if (CUDNN_VERSION >= 91800) + case KnobType_t::SWAP_AB: + cudnn_knob_type = CUDNN_KNOB_TYPE_SWAP_AB; + return cudnnStatus_t::CUDNN_STATUS_SUCCESS; +#endif +#if (CUDNN_VERSION >= 92200) + case KnobType_t::INPUT_TMA_ENABLE: + cudnn_knob_type = CUDNN_KNOB_TYPE_INPUT_TMA_ENABLE; + return cudnnStatus_t::CUDNN_STATUS_SUCCESS; + case KnobType_t::OUTPUT_TMA_ENABLE: + cudnn_knob_type = CUDNN_KNOB_TYPE_OUTPUT_TMA_ENABLE; + return cudnnStatus_t::CUDNN_STATUS_SUCCESS; +#endif +#ifndef NO_DEFAULT_IN_SWITCH + default: + return cudnnStatus_t::CUDNN_STATUS_INVALID_VALUE; +#endif + } + return cudnnStatus_t::CUDNN_STATUS_INVALID_VALUE; +} + +inline KnobType_t +convert_from_backend_knob_type(cudnnBackendKnobType_t cudnn_knob_type) { + switch (cudnn_knob_type) { + case CUDNN_KNOB_TYPE_SWIZZLE: + return KnobType_t::SWIZZLE; + case CUDNN_KNOB_TYPE_TILE_SIZE: + return KnobType_t::TILE_SIZE; + case CUDNN_KNOB_TYPE_EDGE: + return KnobType_t::EDGE; + case CUDNN_KNOB_TYPE_MULTIPLY: + return KnobType_t::MULTIPLY; + case CUDNN_KNOB_TYPE_SPLIT_K_BUF: + return KnobType_t::SPLIT_K_BUF; + case CUDNN_KNOB_TYPE_TILEK: + return KnobType_t::TILEK; + case CUDNN_KNOB_TYPE_STAGES: + return KnobType_t::STAGES; + case CUDNN_KNOB_TYPE_REDUCTION_MODE: + return KnobType_t::REDUCTION_MODE; + case CUDNN_KNOB_TYPE_SPLIT_K_SLC: + return KnobType_t::SPLIT_K_SLC; + case CUDNN_KNOB_TYPE_IDX_MODE: + return KnobType_t::IDX_MODE; + case CUDNN_KNOB_TYPE_SPECFILT: + return KnobType_t::SPECFILT; + case CUDNN_KNOB_TYPE_KERNEL_CFG: + return KnobType_t::KERNEL_CFG; + case CUDNN_KNOB_TYPE_WORKSPACE: + return KnobType_t::WORKSPACE; +#if (CUDNN_VERSION >= 8600) + case CUDNN_KNOB_TYPE_TILE_CGA_M: + return KnobType_t::TILE_CGA_M; + case CUDNN_KNOB_TYPE_TILE_CGA_N: + return KnobType_t::TILE_CGA_N; +#endif +#if (CUDNN_VERSION >= 8800) + case CUDNN_KNOB_TYPE_BLOCK_SIZE: + return KnobType_t::BLOCK_SIZE; +#endif +#if (CUDNN_VERSION >= 8900) + case CUDNN_KNOB_TYPE_OCCUPANCY: + return KnobType_t::OCCUPANCY; + case CUDNN_KNOB_TYPE_ARRAY_SIZE_PER_THREAD: + return KnobType_t::ARRAY_SIZE_PER_THREAD; +#endif +#if (CUDNN_VERSION >= 8905) + case CUDNN_KNOB_TYPE_SPLIT_COLS: + return KnobType_t::SPLIT_COLS; + case CUDNN_KNOB_TYPE_TILE_ROWS: + return KnobType_t::TILE_ROWS; + case CUDNN_KNOB_TYPE_TILE_COLS: + return KnobType_t::TILE_COLS; + case CUDNN_KNOB_TYPE_LOAD_SIZE: + return KnobType_t::LOAD_SIZE; +#endif +#if (CUDNN_VERSION >= 90700) + case CUDNN_KNOB_TYPE_CTA_COUNT: + return KnobType_t::CTA_COUNT; + case CUDNN_KNOB_TYPE_STREAM_K: + return KnobType_t::STREAM_K; + case CUDNN_KNOB_TYPE_SPLIT_P_SLC: + return KnobType_t::SPLIT_P_SLC; + case CUDNN_KNOB_TYPE_TILE_M: + return KnobType_t::TILE_M; + case CUDNN_KNOB_TYPE_TILE_N: + return KnobType_t::TILE_N; + case CUDNN_KNOB_TYPE_WARP_SPEC_CFG: + return KnobType_t::WARP_SPEC_CFG; +#endif +#if (CUDNN_VERSION >= 91800) + case CUDNN_KNOB_TYPE_SWAP_AB: + return KnobType_t::SWAP_AB; +#endif +#if (CUDNN_VERSION >= 92200) + case CUDNN_KNOB_TYPE_INPUT_TMA_ENABLE: + return KnobType_t::INPUT_TMA_ENABLE; + case CUDNN_KNOB_TYPE_OUTPUT_TMA_ENABLE: + return KnobType_t::OUTPUT_TMA_ENABLE; +#endif + default: + return KnobType_t::NOT_SET; + } +} + +} // namespace cudnn_frontend \ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/adaptive_layernorm.h b/deps/cudnn-frontend/include/cudnn_frontend/node/adaptive_layernorm.h new file mode 100644 index 0000000..86eaa6d --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/adaptive_layernorm.h @@ -0,0 +1,454 @@ +#pragma once + +#include "../../cudnn_frontend_Heuristics.h" +#include "../../cudnn_frontend_Logging.h" + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend { + +namespace graph { +class AdaLayerNormNode : public NodeCRTP { + public: + AdaLayernorm_attributes attributes; + + AdaLayerNormNode(AdaLayernorm_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::ADALAYERNORM; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for adalayernorm node " << attributes.name); + + attributes.fill_from_context(context); + + auto X = attributes.inputs[AdaLayernorm_attributes::input_names::X]; + auto Y = attributes.outputs[AdaLayernorm_attributes::output_names::Y]; + + // Only infer dims and strides if user did not set them + if (Y->get_dim().empty()) { + Y->set_dim(X->get_dim()); + } + if (Y->get_stride().empty()) { + Y->set_stride(X->get_stride()); + } + + // scale_bias dim is 1,c,h,w + auto scale_bias_dim = X->get_dim(); + scale_bias_dim[0] = 1; + + auto scale = attributes.inputs[AdaLayernorm_attributes::input_names::SCALE]; + // Only infer dims and strides if user did not set them + if (scale->get_dim().empty()) { + scale->set_dim(scale_bias_dim); + } + if (scale->get_stride().empty()) { + auto const& scale_dim = scale->get_dim(); + std::vector stride_order; + CHECK_CUDNN_FRONTEND_ERROR( + detail::generate_stride_order_preserving_format(X->get_stride(), scale_dim.size(), stride_order)); + scale->set_stride(detail::generate_stride(scale_dim, stride_order)); + } + + auto bias = attributes.inputs[AdaLayernorm_attributes::input_names::BIAS]; + // Only infer dims and strides if user did not set them + if (bias->get_dim().empty()) { + bias->set_dim(scale_bias_dim); + } + if (bias->get_stride().empty()) { + auto const& bias_dim = bias->get_dim(); + std::vector stride_order; + CHECK_CUDNN_FRONTEND_ERROR( + detail::generate_stride_order_preserving_format(X->get_stride(), bias_dim.size(), stride_order)); + bias->set_stride(detail::generate_stride(bias_dim, stride_order)); + } + + if (attributes.forward_phase == NormFwdPhase_t::TRAINING) { + // stats dim is x where scale == 1 else 1 + auto stats_dim = X->get_dim(); + for (size_t i = 1; i < stats_dim.size(); i++) { + if (scale->get_dim()[i] != 1) { + stats_dim[i] = 1; + } + } + + auto mean = attributes.outputs[AdaLayernorm_attributes::output_names::MEAN]; + // Only infer dims and strides if user did not set them + if (mean->get_dim().empty()) { + mean->set_dim(stats_dim); + } + if (mean->get_stride().empty()) { + auto const& mean_dim = mean->get_dim(); + std::vector stride_order; + CHECK_CUDNN_FRONTEND_ERROR( + detail::generate_stride_order_preserving_format(X->get_stride(), mean_dim.size(), stride_order)); + mean->set_stride(detail::generate_stride(mean_dim, stride_order)); + } + + auto inv_var = attributes.outputs[AdaLayernorm_attributes::output_names::INV_VARIANCE]; + // Only infer dims and strides if user did not set them + if (inv_var->get_dim().empty()) { + inv_var->set_dim(stats_dim); + } + if (inv_var->get_stride().empty()) { + auto const& inv_var_dim = inv_var->get_dim(); + std::vector stride_order; + CHECK_CUDNN_FRONTEND_ERROR( + detail::generate_stride_order_preserving_format(X->get_stride(), inv_var_dim.size(), stride_order)); + inv_var->set_stride(detail::generate_stride(inv_var_dim, stride_order)); + } + } + + // Set scalar tensors + std::vector ones(X->get_dim().size(), 1); + auto infer_scalar_tensors = [&ones](std::shared_ptr& T) { + // Only infer dims and strides if user did not set them + if (T->get_dim().empty()) { + T->set_dim(ones); + } + if (T->get_stride().empty()) { + T->set_stride(ones); + } + }; + infer_scalar_tensors(attributes.inputs[AdaLayernorm_attributes::input_names::EPSILON]); + + return {error_code_t::OK, ""}; + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: " << "Validating AdaLayerNormNode " << attributes.name); + // Norm forward phase should be set + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.forward_phase == NormFwdPhase_t::NOT_SET, + error_code_t::ATTRIBUTE_NOT_SET, + "Forward phase not set of adalayernorm node."); + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set& uids_involved_in_operations, + std::vector>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map>& tensors) const override final { + getLogger() << "[cudnn_frontend] INFO: " << "Building AdaLayernorm operations " << attributes.name << std::endl; + + auto cudnn_ver_error = error_t{error_code_t::GRAPH_NOT_SUPPORTED, "AdaLN fwd requires cuDNN v9.9.0"}; +#if (CUDNN_VERSION >= 90900) + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(90900, cudnn_ver_error); + CUDNN_FRONTEND_UNUSED(operations); + auto adalayernorm_operation = + make_shared_backend_pointer((cudnnBackendDescriptorType_t)CUDNN_BACKEND_OPERATION_NORM_FORWARD_DESCRIPTOR); + + cudnnBackendNormMode_t cudnn_norm_mode; + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormMode_t::ADA_LAYER_NORM, cudnn_norm_mode)); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_NORM_FWD_MODE, + CUDNN_TYPE_NORM_MODE, + 1, + &cudnn_norm_mode)); + + cudnnBackendNormFwdPhase_t cudnn_norm_fwd_phase; + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.forward_phase, cudnn_norm_fwd_phase)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_NORM_FWD_PHASE, + CUDNN_TYPE_NORM_FWD_PHASE, + 1, + &cudnn_norm_fwd_phase)); + + auto X = attributes.inputs.find(AdaLayernorm_attributes::input_names::X)->second; + auto backend_x = tensors[X->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_NORM_FWD_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_x)); + + auto Scale = attributes.inputs.find(AdaLayernorm_attributes::input_names::SCALE)->second; + auto backend_scale = tensors[Scale->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_NORM_FWD_SCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_scale)); + + auto Bias_iter = attributes.inputs.find(AdaLayernorm_attributes::input_names::BIAS); + if (Bias_iter != attributes.inputs.end() && Bias_iter->second->get_is_virtual() == false) { + auto backend_bias = tensors[Bias_iter->second->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_NORM_FWD_BIAS_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_bias)); + } + + auto Epsilon = attributes.inputs.find(AdaLayernorm_attributes::input_names::EPSILON)->second; + auto backend_epsilon = tensors[Epsilon->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_NORM_FWD_EPSILON_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_epsilon)); + + auto Y = attributes.outputs.find(AdaLayernorm_attributes::output_names::Y)->second; + auto backend_y = tensors[Y->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_NORM_FWD_YDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_y)); + + if (attributes.forward_phase == NormFwdPhase_t::TRAINING) { + auto Mean = attributes.outputs.find(AdaLayernorm_attributes::output_names::MEAN)->second; + auto backend_mean = tensors[Mean->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_NORM_FWD_MEAN_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_mean)); + + auto Inv_variance = attributes.outputs.find(AdaLayernorm_attributes::output_names::INV_VARIANCE)->second; + auto backend_inv_variance = tensors[Inv_variance->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_NORM_FWD_INV_VARIANCE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_inv_variance)); + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(adalayernorm_operation->get_backend_descriptor())); + + raw_operations.push_back(adalayernorm_operation); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; +#else + CUDNN_FRONTEND_UNUSED(uids_involved_in_operations); + CUDNN_FRONTEND_UNUSED(operations); + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FRONTEND_UNUSED(tensors); + return cudnn_ver_error; +#endif + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "ADA_LAYER_NORM"})"_json); + } +#endif +}; + +/*******/ + +class DAdaLayerNormNode : public NodeCRTP { + public: + AdaLayernorm_backward_attributes attributes; + + DAdaLayerNormNode(AdaLayernorm_backward_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::DADALAYERNORM; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for DAdaLayerNorm node " << attributes.name); + + attributes.fill_from_context(context); + + // TODO: Only inferencing from X works today. + auto X = attributes.inputs[AdaLayernorm_backward_attributes::input_names::X]; + auto const x_tensor_dim = X->get_dim(); + + auto DY = attributes.inputs[AdaLayernorm_backward_attributes::input_names::DY]; + auto dy_tensor_dim = DY->get_dim(); + + // Only infer dims and strides if user did not set them + if (dy_tensor_dim.empty()) { + dy_tensor_dim.resize(x_tensor_dim.size()); + DY->set_dim(x_tensor_dim); + } + if (DY->get_stride().empty()) { + auto const& DY_dim = DY->get_dim(); + // Default to NCHW + auto const& stride_order = detail::generate_row_major_stride_order(DY_dim.size()); + DY->set_stride(detail::generate_stride(DY_dim, stride_order)); + } + + auto DX = attributes.outputs[AdaLayernorm_backward_attributes::output_names::DX]; + auto dx_tensor_dim = DX->get_dim(); + // Only infer dims and strides if user did not set them + if (dx_tensor_dim.empty()) { + dx_tensor_dim.resize(x_tensor_dim.size()); + DX->set_dim(x_tensor_dim); + } + if (DX->get_stride().empty()) { + auto const& DX_dim = DX->get_dim(); + // Default to NCHW + auto const& stride_order = detail::generate_row_major_stride_order(DX_dim.size()); + DX->set_stride(detail::generate_stride(DX_dim, stride_order)); + } + + auto SCALE = attributes.inputs[AdaLayernorm_backward_attributes::input_names::SCALE]; + auto scale_bias_dim = SCALE->get_dim(); + + // Set channel length tensors + auto infer_scale_bias_tensors = [&scale_bias_dim](std::shared_ptr& T) { + auto tensor_dim = T->get_dim(); + // Only infer dims and strides if user did not set them + if (tensor_dim.empty()) { + T->set_dim(scale_bias_dim); + } + if (T->get_stride().empty()) { + auto const& T_dim = T->get_dim(); + // Default to NCHW + auto const& stride_order = detail::generate_row_major_stride_order(T_dim.size()); + T->set_stride(detail::generate_stride(T_dim, stride_order)); + } + }; + + infer_scale_bias_tensors(attributes.outputs[AdaLayernorm_backward_attributes::output_names::DSCALE]); + auto DBIAS = attributes.outputs.at(AdaLayernorm_backward_attributes::output_names::DBIAS); + if (DBIAS->get_is_virtual() == false) { + infer_scale_bias_tensors(DBIAS); + } + + return {error_code_t::OK, ""}; + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating DAdaLayerNormNode node " << attributes.name); + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set& uids_involved_in_operations, + std::vector>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map>& tensors) const override final { + getLogger() << "[cudnn_frontend] INFO: " << "Building DAdaLayerNormNode operations " << attributes.name + << std::endl; + auto cudnn_ver_error = error_t{error_code_t::GRAPH_NOT_SUPPORTED, "AdaLN bwd requires cuDNN v9.9.0"}; +#if (CUDNN_VERSION >= 90900) + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(90900, cudnn_ver_error); + CUDNN_FRONTEND_UNUSED(operations); + auto adalayernorm_operation = + make_shared_backend_pointer((cudnnBackendDescriptorType_t)CUDNN_BACKEND_OPERATION_NORM_BACKWARD_DESCRIPTOR); + + cudnnBackendNormMode_t cudnn_norm_mode; + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormMode_t::ADA_LAYER_NORM, cudnn_norm_mode)); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_NORM_BWD_MODE, + CUDNN_TYPE_NORM_MODE, + 1, + &cudnn_norm_mode)); + + auto X = attributes.inputs.find(AdaLayernorm_backward_attributes::input_names::X)->second; + auto backend_x = tensors[X->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_NORM_BWD_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_x)); + + auto Mean = attributes.inputs.find(AdaLayernorm_backward_attributes::input_names::MEAN)->second; + auto backend_mean = tensors[Mean->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_NORM_BWD_MEAN_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_mean)); + + auto Inv_variance = attributes.inputs.find(AdaLayernorm_backward_attributes::input_names::INV_VARIANCE)->second; + auto backend_inv_variance = tensors[Inv_variance->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_NORM_BWD_INV_VARIANCE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_inv_variance)); + + auto Dy = attributes.inputs.find(AdaLayernorm_backward_attributes::input_names::DY)->second; + auto backend_dy = tensors[Dy->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_NORM_BWD_DYDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_dy)); + + auto Scale = attributes.inputs.find(AdaLayernorm_backward_attributes::input_names::SCALE)->second; + auto backend_scale = tensors[Scale->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_NORM_BWD_SCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_scale)); + + auto Dx = attributes.outputs.find(AdaLayernorm_backward_attributes::output_names::DX)->second; + auto backend_dx = tensors[Dx->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_NORM_BWD_DXDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_dx)); + + auto Dscale = attributes.outputs.find(AdaLayernorm_backward_attributes::output_names::DSCALE)->second; + auto backend_dscale = tensors[Dscale->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_NORM_BWD_DSCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_dscale)); + + auto Dbias_iter = attributes.outputs.find(AdaLayernorm_backward_attributes::output_names::DBIAS); + if (Dbias_iter != attributes.outputs.end() && Dbias_iter->second->get_is_virtual() == false) { + auto backend_dbias = tensors[Dbias_iter->second->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(adalayernorm_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_NORM_BWD_DBIAS_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_dbias)); + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(adalayernorm_operation->get_backend_descriptor())); + + raw_operations.push_back(adalayernorm_operation); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; +#else + CUDNN_FRONTEND_UNUSED(uids_involved_in_operations); + CUDNN_FRONTEND_UNUSED(operations); + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FRONTEND_UNUSED(tensors); + return cudnn_ver_error; +#endif + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "ADA_LAYER_NORM_BPROP"})"_json); + } +#endif +}; + +} // namespace graph + +} // namespace cudnn_frontend \ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/batchnorm.h b/deps/cudnn-frontend/include/cudnn_frontend/node/batchnorm.h new file mode 100644 index 0000000..69d232b --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/batchnorm.h @@ -0,0 +1,268 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend { + +namespace graph { +class BatchNormNode : public NodeCRTP { + public: + Batchnorm_attributes attributes; + + BatchNormNode(Batchnorm_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::BATCHNORM; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for batchnorm node " << attributes.name); + + attributes.fill_from_context(context); + + auto X = attributes.inputs[Batchnorm_attributes::input_names::X]; + auto Y = attributes.outputs[Batchnorm_attributes::output_names::Y]; + // Only infer dims and strides if user did not set them + if (Y->get_dim().empty()) { + Y->set_dim(X->get_dim()); + } + if (Y->get_stride().empty()) { + Y->set_stride(X->get_stride()); + } + + // Set channel length tensors + auto const x_tensor_dim = X->get_dim(); + auto infer_per_channel_tensors = [&x_tensor_dim](std::shared_ptr& T) { + auto tensor_dim = T->get_dim(); + // Only infer dims and strides if user did not set them + if (tensor_dim.empty()) { + tensor_dim.resize(x_tensor_dim.size(), 1); + tensor_dim[1] = x_tensor_dim[1]; + T->set_dim(tensor_dim); + } + if (T->get_stride().empty()) { + auto const& T_dim = T->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(T_dim.size()); + T->set_stride(detail::generate_stride(T_dim, stride_order)); + } + }; + infer_per_channel_tensors(attributes.outputs[Batchnorm_attributes::output_names::MEAN]); + infer_per_channel_tensors(attributes.outputs[Batchnorm_attributes::output_names::INV_VARIANCE]); + + auto has_running_stats = attributes.inputs[Batchnorm_attributes::input_names::PREV_RUNNING_MEAN] || + attributes.inputs[Batchnorm_attributes::input_names::PREV_RUNNING_VAR]; + + if (has_running_stats) { + infer_per_channel_tensors(attributes.outputs[Batchnorm_attributes::output_names::NEXT_RUNNING_MEAN]); + infer_per_channel_tensors(attributes.outputs[Batchnorm_attributes::output_names::NEXT_RUNNING_VAR]); + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set& uids_involved_in_operations, + std::vector>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: Building BatchNormNode operations " << attributes.name << " "); + + // Create operation by directly calling cuDNN backend API + Operation_v8 batchnorm_operation; + + _CUDNN_CHECK_CUDNN_ERROR( + batchnorm_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_NORM_FORWARD_DESCRIPTOR)); + + // Set norm mode to BATCH_NORM + cudnnBackendNormMode_t cudnn_norm_mode; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormMode_t::BATCH_NORM, cudnn_norm_mode)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_MODE, + CUDNN_TYPE_NORM_MODE, + 1, + &cudnn_norm_mode)); + + // Set forward phase to TRAINING + cudnnBackendNormFwdPhase_t cudnn_norm_fwd_phase; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormFwdPhase_t::TRAINING, cudnn_norm_fwd_phase)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_PHASE, + CUDNN_TYPE_NORM_FWD_PHASE, + 1, + &cudnn_norm_fwd_phase)); + + // Set input tensor X + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Batchnorm_attributes::input_names::X); + auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &x_desc)); + + // Set saved mean and inv_variance + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(MEAN, Batchnorm_attributes::output_names::MEAN); + auto mean_desc = tensors.at(MEAN->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_MEAN_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &mean_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(INV_VARIANCE, Batchnorm_attributes::output_names::INV_VARIANCE); + auto inv_var_desc = tensors.at(INV_VARIANCE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_INV_VARIANCE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &inv_var_desc)); + + // Set scale and bias tensors + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SCALE, Batchnorm_attributes::input_names::SCALE); + auto scale_desc = tensors.at(SCALE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_SCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &scale_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(BIAS, Batchnorm_attributes::input_names::BIAS); + auto bias_desc = tensors.at(BIAS->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_BIAS_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &bias_desc)); + + // Set epsilon tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(EPSILON, Batchnorm_attributes::input_names::EPSILON); + auto epsilon_desc = tensors.at(EPSILON->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_EPSILON_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &epsilon_desc)); + + // Check for running stats + bool has_running_stats = true; + auto it = attributes.inputs.find(Batchnorm_attributes::input_names::PREV_RUNNING_MEAN); + if (it == attributes.inputs.end() || it->second == nullptr) { + has_running_stats = false; + } + + if (has_running_stats) { + // Set momentum (exp decay factor) + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(MOMENTUM, Batchnorm_attributes::input_names::MOMENTUM); + auto momentum_desc = tensors.at(MOMENTUM->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_EXP_AVG_FACTOR_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &momentum_desc)); + + // Set prev running mean and var + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(PREV_RUNNING_MEAN, + Batchnorm_attributes::input_names::PREV_RUNNING_MEAN); + auto prev_mean_desc = tensors.at(PREV_RUNNING_MEAN->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_INPUT_RUNNING_MEAN_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &prev_mean_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(PREV_RUNNING_VAR, + Batchnorm_attributes::input_names::PREV_RUNNING_VAR); + auto prev_var_desc = tensors.at(PREV_RUNNING_VAR->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_INPUT_RUNNING_VAR_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &prev_var_desc)); + + // Set next running mean and var + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(NEXT_RUNNING_MEAN, + Batchnorm_attributes::output_names::NEXT_RUNNING_MEAN); + auto next_mean_desc = tensors.at(NEXT_RUNNING_MEAN->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_OUTPUT_RUNNING_MEAN_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &next_mean_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(NEXT_RUNNING_VAR, + Batchnorm_attributes::output_names::NEXT_RUNNING_VAR); + auto next_var_desc = tensors.at(NEXT_RUNNING_VAR->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_OUTPUT_RUNNING_VAR_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &next_var_desc)); + } + + // Set output tensor Y + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(Y, Batchnorm_attributes::output_names::Y); + auto y_desc = tensors.at(Y->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_YDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &y_desc)); + + // Set peer stat tensors if any + if (!attributes.peer_stats.empty()) { + std::vector peer_stat_descs; + for (auto const& peer_stat : attributes.peer_stats) { + peer_stat_descs.push_back(tensors.at(peer_stat->get_uid())->get_raw_desc()); + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_PEER_STAT_DESCS, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + peer_stat_descs.size(), + peer_stat_descs.data())); + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(batchnorm_operation.get_raw_desc())); + + operations.push_back(std::make_shared(std::move(batchnorm_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "BATCHNORM"})"_json); + } +#endif +}; + +} // namespace graph + +} // namespace cudnn_frontend \ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/batchnorm_inference.h b/deps/cudnn-frontend/include/cudnn_frontend/node/batchnorm_inference.h new file mode 100644 index 0000000..9fb433f --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/batchnorm_inference.h @@ -0,0 +1,156 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend { + +namespace graph { +class BatchnormInferenceNode : public NodeCRTP { + public: + Batchnorm_inference_attributes attributes; + + BatchnormInferenceNode(Batchnorm_inference_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::BATCHNORM_INFERENCE; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for batchnorm inference node " << attributes.name); + + attributes.fill_from_context(context); + + auto X = attributes.inputs[Batchnorm_inference_attributes::input_names::X]; + auto Y = attributes.outputs[Batchnorm_inference_attributes::output_names::Y]; + // Only infer dims and strides if user did not set them + if (Y->get_dim().empty()) { + Y->set_dim(X->get_dim()); + } + if (Y->get_stride().empty()) { + Y->set_stride(X->get_stride()); + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set& uids_involved_in_operations, + std::vector>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: Building BatchnormInferenceNode operations " << attributes.name << " "); + + // Create operation by directly calling cuDNN backend API + Operation_v8 batchnorm_operation; + + _CUDNN_CHECK_CUDNN_ERROR( + batchnorm_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_NORM_FORWARD_DESCRIPTOR)); + + // Set norm mode to BATCH_NORM + cudnnBackendNormMode_t cudnn_norm_mode; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormMode_t::BATCH_NORM, cudnn_norm_mode)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_MODE, + CUDNN_TYPE_NORM_MODE, + 1, + &cudnn_norm_mode)); + + // Set forward phase to INFERENCE + cudnnBackendNormFwdPhase_t cudnn_norm_fwd_phase; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormFwdPhase_t::INFERENCE, cudnn_norm_fwd_phase)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_PHASE, + CUDNN_TYPE_NORM_FWD_PHASE, + 1, + &cudnn_norm_fwd_phase)); + + // Set input tensor X + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Batchnorm_inference_attributes::input_names::X); + auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &x_desc)); + + // Set mean and inv_variance (as inputs for inference) + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(MEAN, Batchnorm_inference_attributes::input_names::MEAN); + auto mean_desc = tensors.at(MEAN->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_MEAN_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &mean_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(INV_VARIANCE, + Batchnorm_inference_attributes::input_names::INV_VARIANCE); + auto inv_var_desc = tensors.at(INV_VARIANCE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_INV_VARIANCE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &inv_var_desc)); + + // Set scale and bias tensors + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SCALE, Batchnorm_inference_attributes::input_names::SCALE); + auto scale_desc = tensors.at(SCALE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_SCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &scale_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(BIAS, Batchnorm_inference_attributes::input_names::BIAS); + auto bias_desc = tensors.at(BIAS->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_BIAS_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &bias_desc)); + + // Set output tensor Y + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(Y, Batchnorm_inference_attributes::output_names::Y); + auto y_desc = tensors.at(Y->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(batchnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_YDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &y_desc)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(batchnorm_operation.get_raw_desc())); + + operations.push_back(std::make_shared(std::move(batchnorm_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "BATCHNORM_INFERENCE"})"_json); + } +#endif +}; + +} // namespace graph + +} // namespace cudnn_frontend \ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/block_scale_dequantize.h b/deps/cudnn-frontend/include/cudnn_frontend/node/block_scale_dequantize.h new file mode 100644 index 0000000..bbaa2bf --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/block_scale_dequantize.h @@ -0,0 +1,169 @@ +#pragma once + +#include "../../cudnn_frontend_Logging.h" +#include "../../cudnn_frontend_shim.h" + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend { + +namespace graph { + +class BlockScaleDequantizeNode : public NodeCRTP { + public: + Block_scale_dequantize_attributes attributes; + + BlockScaleDequantizeNode(Block_scale_dequantize_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::BLOCK_SCALE_DEQUANTIZE; + } + + error_t + pre_validate_node() const override final { + getLogger() << "[cudnn_frontend] INFO: " + << "Validating BlockScaleDequantizeNode " << attributes.name << std::endl; + + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.block_size.empty(), error_code_t::ATTRIBUTE_NOT_SET, "Block size not set\n"); + + auto Y = attributes.outputs.at(Block_scale_dequantize_attributes::output_names::Y); + + RETURN_CUDNN_FRONTEND_ERROR_IF(!(Y->get_is_virtual()), + error_code_t::INVALID_VALUE, + "Output tensor of dequantize node should be virtual\n"); + + return {error_code_t::OK, ""}; + } + + error_t + infer_properties_node() override final { + getLogger() << "[cudnn_frontend] INFO: Inferencing properties for BlockScaleDequantizeNode " << attributes.name + << std::endl; + + attributes.fill_from_context(context); + + auto X = attributes.inputs[Block_scale_dequantize_attributes::input_names::X]; + auto scale = attributes.inputs[Block_scale_dequantize_attributes::input_names::scale]; + auto Y = attributes.outputs[Block_scale_dequantize_attributes::output_names::Y]; + + // Only infer dims and strides if user did not set them + if (Y->get_dim().empty()) { + Y->set_dim(X->get_dim()); + } + + if (Y->get_stride().empty()) { + Y->set_stride(X->get_stride()); + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set& uids_involved_in_operations, + std::vector>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map>& tensors) const override final { + getLogger() << "[cudnn_frontend] INFO: " + << "Building BlockScaleDequantizeNode operations " << attributes.name << std::endl; + auto cudnn_ver_error = + error_t{error_code_t::GRAPH_NOT_SUPPORTED, "Block scale dequantize requires cuDNN v9.7.0"}; + +#if (CUDNN_VERSION >= 90700) // TODO: v9.99 is new feature branch; switch to release branch when ready + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(90700, cudnn_ver_error); + CUDNN_FRONTEND_UNUSED(operations); + auto block_scale_dequantize_operation = make_shared_backend_pointer( + (cudnnBackendDescriptorType_t)CUDNN_BACKEND_OPERATION_BLOCK_SCALE_DEQUANTIZE_DESCRIPTOR); + + auto X = attributes.inputs.find(Block_scale_dequantize_attributes::input_names::X)->second; + auto backend_x = tensors[X->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(block_scale_dequantize_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_BLOCK_SCALE_DEQUANTIZE_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_x)); + + auto scale = attributes.inputs.find(Block_scale_dequantize_attributes::input_names::scale)->second; + auto backend_scale = tensors[scale->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(block_scale_dequantize_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_BLOCK_SCALE_DEQUANTIZE_SCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_scale)); + + auto Y = attributes.outputs.find(Block_scale_dequantize_attributes::output_names::Y)->second; + auto backend_y = tensors[Y->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(block_scale_dequantize_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_BLOCK_SCALE_DEQUANTIZE_YDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_y)); + + cudnnDataType_t cudnn_data_type; + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.compute_data_type, cudnn_data_type)); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(block_scale_dequantize_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_BLOCK_SCALE_DEQUANTIZE_MATH_PREC, + CUDNN_TYPE_DATA_TYPE, + 1, + &cudnn_data_type)); + + const int32_t* block_size = attributes.block_size.data(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(block_scale_dequantize_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_BLOCK_SCALE_DEQUANTIZE_BLOCK_SIZE, + CUDNN_TYPE_INT32, + attributes.block_size.size(), + block_size)); + +#if (CUDNN_VERSION >= 91400) + if (detail::get_backend_version() >= 91400) { + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(block_scale_dequantize_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_BLOCK_SCALE_DEQUANTIZE_NEG_SCALE, + CUDNN_TYPE_BOOLEAN, + 1, + &attributes.is_negative_scale)); + } +#endif + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(block_scale_dequantize_operation->get_backend_descriptor())); + + raw_operations.push_back(block_scale_dequantize_operation); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; +#else + CUDNN_FRONTEND_UNUSED(uids_involved_in_operations); + CUDNN_FRONTEND_UNUSED(operations); + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FRONTEND_UNUSED(tensors); + return cudnn_ver_error; +#endif + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "BLOCK_SCALE_DEQUANTIZE"})"_json); + } +#endif +}; + +inline void +INode::block_scale_dequantize(std::shared_ptr x, + std::shared_ptr scale, + Block_scale_dequantize_attributes attributes, + std::shared_ptr y) { + attributes.inputs[Block_scale_dequantize_attributes::input_names::X] = x; + attributes.inputs[Block_scale_dequantize_attributes::input_names::scale] = scale; + attributes.outputs[Block_scale_dequantize_attributes::output_names::Y] = y; + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); +} + +} // namespace graph + +} // namespace cudnn_frontend \ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/block_scale_quantize.h b/deps/cudnn-frontend/include/cudnn_frontend/node/block_scale_quantize.h new file mode 100644 index 0000000..494ea14 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/block_scale_quantize.h @@ -0,0 +1,205 @@ +#pragma once + +#include "../../cudnn_frontend_Logging.h" +#include "../../cudnn_frontend_shim.h" + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend { + +namespace graph { + +class BlockScaleQuantizeNode : public NodeCRTP { + public: + Block_scale_quantize_attributes attributes; + + BlockScaleQuantizeNode(Block_scale_quantize_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::BLOCK_SCALE_QUANTIZE; + } + + error_t + pre_validate_node() const override final { + getLogger() << "[cudnn_frontend] INFO: " << "Validating BlockScaleQuantizeNode " << attributes.name + << std::endl; + + RETURN_CUDNN_FRONTEND_ERROR_IF( + !attributes.block_size.has_value(), error_code_t::ATTRIBUTE_NOT_SET, "Block size not set."); + + return {error_code_t::OK, ""}; + } + + error_t + infer_properties_node() override final { + getLogger() << "[cudnn_frontend] INFO: Inferencing properties for BlockScaleQuantizeNode " << attributes.name + << std::endl; + + attributes.fill_from_context(context); + + auto X = attributes.inputs[Block_scale_quantize_attributes::input_names::X]; + auto Y = attributes.outputs[Block_scale_quantize_attributes::output_names::Y]; + auto scale = attributes.outputs[Block_scale_quantize_attributes::output_names::scale]; + + // Block scale quantize requires the block scale axis to be packed + auto infer_strides_transposed = [&X](std::shared_ptr& T, + std::optional const& axis) { + auto const& dim = T->get_dim(); + auto const& X_dim = X->get_dim(); + auto const& X_stride = X->get_stride(); + + std::vector indices(X_stride.size()); + std::iota(indices.begin(), indices.end(), 0); + // Sort indices based on stride values in descending order + std::sort(indices.begin(), indices.end(), [&X_dim, &X_stride](int64_t i, int64_t j) { + // Prioritize singleton dimensions + if (X_stride[i] == X_stride[j]) { + return (X_dim[i] == 1) && (X_dim[j] != 1); + } + return X_stride[i] < X_stride[j]; + }); + if (axis) { + // Rotate left until the axis is the packed dim + std::rotate(indices.begin(), std::find(indices.begin(), indices.end(), axis.value()), indices.end()); + } + std::vector stride_order(X_stride.size()); + for (size_t i = 0; i < indices.size(); ++i) { + stride_order[indices[i]] = i; + } + T->set_stride(detail::generate_stride(dim, stride_order)); + }; + + // Only infer dims and strides if user did not set them + if (Y->get_dim().empty()) { + Y->set_dim(X->get_dim()); + } + if (Y->get_stride().empty()) { + if (attributes.transpose) { + infer_strides_transposed(Y, attributes.axis); + } else { + Y->set_stride(X->get_stride()); + } + } + + // Only infer dims and strides if user did not set them + if (scale->get_dim().empty()) { + auto scale_dim = X->get_dim(); + if (attributes.axis) { + scale_dim[attributes.axis.value()] /= attributes.block_size.value(); + } else { + scale_dim.back() /= attributes.block_size.value(); + } + scale->set_dim(scale_dim); + } + if (scale->get_stride().empty()) { + if (attributes.transpose) { + infer_strides_transposed(scale, attributes.axis); + } else { + auto const& scale_dim = scale->get_dim(); + std::vector stride_order; + CHECK_CUDNN_FRONTEND_ERROR( + detail::generate_stride_order_preserving_format(X->get_stride(), scale_dim.size(), stride_order)); + scale->set_stride(detail::generate_stride(scale_dim, stride_order)); + } + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set& uids_involved_in_operations, + std::vector>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map>& tensors) const override final { + getLogger() << "[cudnn_frontend] INFO: " << "Building BlockScaleQuantizeNode operations " << attributes.name + << std::endl; + auto cudnn_ver_error = error_t{error_code_t::GRAPH_NOT_SUPPORTED, "Block scale quantize requires cuDNN v9.7.0"}; + +#if (CUDNN_VERSION >= 90700) // TODO: v9.99 is new feature branch; switch to release branch when ready + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(90700, cudnn_ver_error); + CUDNN_FRONTEND_UNUSED(operations); + auto block_scale_quantize_operation = make_shared_backend_pointer( + (cudnnBackendDescriptorType_t)CUDNN_BACKEND_OPERATION_BLOCK_SCALE_QUANTIZE_DESCRIPTOR); + + auto X = attributes.inputs.find(Block_scale_quantize_attributes::input_names::X)->second; + auto backend_x = tensors[X->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(block_scale_quantize_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_BLOCK_SCALE_QUANTIZE_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_x)); + + auto Y = attributes.outputs.find(Block_scale_quantize_attributes::output_names::Y)->second; + auto backend_y = tensors[Y->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(block_scale_quantize_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_BLOCK_SCALE_QUANTIZE_YDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_y)); + + auto scale = attributes.outputs.find(Block_scale_quantize_attributes::output_names::scale)->second; + auto backend_scale = tensors[scale->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(block_scale_quantize_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_BLOCK_SCALE_QUANTIZE_SCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_scale)); + + cudnnDataType_t cudnn_data_type; + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.compute_data_type, cudnn_data_type)); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(block_scale_quantize_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_BLOCK_SCALE_QUANTIZE_MATH_PREC, + CUDNN_TYPE_DATA_TYPE, + 1, + &cudnn_data_type)); + + int32_t block_size = attributes.block_size.value(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(block_scale_quantize_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_BLOCK_SCALE_QUANTIZE_BLOCK_SIZE, + CUDNN_TYPE_INT32, + 1, + &block_size)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(block_scale_quantize_operation->get_backend_descriptor())); + + raw_operations.push_back(block_scale_quantize_operation); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; +#else + CUDNN_FRONTEND_UNUSED(uids_involved_in_operations); + CUDNN_FRONTEND_UNUSED(operations); + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FRONTEND_UNUSED(tensors); + return cudnn_ver_error; +#endif + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "BLOCK_SCALE_QUANTIZE"})"_json); + } +#endif +}; + +inline void +INode::block_scale_quantize(std::shared_ptr x, + Block_scale_quantize_attributes attributes, + std::shared_ptr y, + std::shared_ptr scale) { + attributes.inputs[Block_scale_quantize_attributes::input_names::X] = x; + attributes.outputs[Block_scale_quantize_attributes::output_names::Y] = y; + attributes.outputs[Block_scale_quantize_attributes::output_names::scale] = scale; + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); +} + +} // namespace graph + +} // namespace cudnn_frontend diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/bn_finalize.h b/deps/cudnn-frontend/include/cudnn_frontend/node/bn_finalize.h new file mode 100644 index 0000000..99d3ec0 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/bn_finalize.h @@ -0,0 +1,263 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend { + +namespace graph { + +class BatchNormFinalizeNode : public NodeCRTP { + public: + BN_finalize_attributes attributes; + + BatchNormFinalizeNode(BN_finalize_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::BN_FINALIZE; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO:Inferencing properties for batchnorm finalize node " << attributes.name); + + attributes.fill_from_context(context); + + auto SUM = attributes.inputs[BN_finalize_attributes::input_names::SUM]; + auto const sum_tensor_dim = SUM->get_dim(); + + // Set channel length tensors + auto infer_per_channel_tensors = [&sum_tensor_dim](std::shared_ptr& T) { + auto tensor_dim = T->get_dim(); + // Only infer dims and strides if user did not set them + if (tensor_dim.empty()) { + tensor_dim = sum_tensor_dim; + T->set_dim(tensor_dim); + } + if (T->get_stride().empty()) { + auto const& T_dim = T->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(T_dim.size()); + T->set_stride(detail::generate_stride(T_dim, stride_order)); + } + }; + infer_per_channel_tensors(attributes.outputs[BN_finalize_attributes::output_names::EQ_BIAS]); + infer_per_channel_tensors(attributes.outputs[BN_finalize_attributes::output_names::EQ_SCALE]); + infer_per_channel_tensors(attributes.outputs[BN_finalize_attributes::output_names::MEAN]); + infer_per_channel_tensors(attributes.outputs[BN_finalize_attributes::output_names::INV_VARIANCE]); + infer_per_channel_tensors(attributes.outputs[BN_finalize_attributes::output_names::NEXT_RUNNING_MEAN]); + infer_per_channel_tensors(attributes.outputs[BN_finalize_attributes::output_names::NEXT_RUNNING_VAR]); + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set& uids_involved_in_operations, + std::vector>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: " << "Building BatchNormFinalizeNode operations " << attributes.name << " "); + + // Create operation by directly calling cuDNN backend API + Operation_v8 bn_finalize_operation; + + _CUDNN_CHECK_CUDNN_ERROR(bn_finalize_operation.initialize_managed_backend_pointer( + CUDNN_BACKEND_OPERATION_BN_FINALIZE_STATISTICS_DESCRIPTOR)); + + // Set BN finalize mode + cudnnBnFinalizeStatsMode_t bn_finalize_mode = CUDNN_BN_FINALIZE_STATISTICS_TRAINING; + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_FINALIZE_STATS_MODE, + CUDNN_TYPE_BN_FINALIZE_STATS_MODE, + 1, + &bn_finalize_mode)); + + // Set compute type (math precision) + cudnnDataType_t compute_type = CUDNN_DATA_FLOAT; + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_FINALIZE_MATH_PREC, + CUDNN_TYPE_DATA_TYPE, + 1, + &compute_type)); + + // Set SUM input tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SUM, BN_finalize_attributes::input_names::SUM); + auto sum_desc = tensors.at(SUM->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_FINALIZE_Y_SUM_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &sum_desc)); + + // Set SQ_SUM input tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SQ_SUM, BN_finalize_attributes::input_names::SQ_SUM); + auto sq_sum_desc = tensors.at(SQ_SUM->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_FINALIZE_Y_SQ_SUM_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &sq_sum_desc)); + + // Set SCALE input tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SCALE, BN_finalize_attributes::input_names::SCALE); + auto scale_desc = tensors.at(SCALE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_FINALIZE_SCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &scale_desc)); + + // Set BIAS input tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(BIAS, BN_finalize_attributes::input_names::BIAS); + auto bias_desc = tensors.at(BIAS->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_FINALIZE_BIAS_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &bias_desc)); + + // Set EQ_SCALE output tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(EQ_SCALE, BN_finalize_attributes::output_names::EQ_SCALE); + auto eq_scale_desc = tensors.at(EQ_SCALE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_FINALIZE_EQ_SCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &eq_scale_desc)); + + // Set EQ_BIAS output tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(EQ_BIAS, BN_finalize_attributes::output_names::EQ_BIAS); + auto eq_bias_desc = tensors.at(EQ_BIAS->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_FINALIZE_EQ_BIAS_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &eq_bias_desc)); + + // Set PREV_RUNNING_MEAN input tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(PREV_RUNNING_MEAN, + BN_finalize_attributes::input_names::PREV_RUNNING_MEAN); + auto prev_running_mean_desc = tensors.at(PREV_RUNNING_MEAN->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_FINALIZE_PREV_RUNNING_MEAN_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &prev_running_mean_desc)); + + // Set PREV_RUNNING_VAR input tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(PREV_RUNNING_VAR, + BN_finalize_attributes::input_names::PREV_RUNNING_VAR); + auto prev_running_var_desc = tensors.at(PREV_RUNNING_VAR->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_FINALIZE_PREV_RUNNING_VAR_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &prev_running_var_desc)); + + // Set NEXT_RUNNING_MEAN output tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(NEXT_RUNNING_MEAN, + BN_finalize_attributes::output_names::NEXT_RUNNING_MEAN); + auto next_running_mean_desc = tensors.at(NEXT_RUNNING_MEAN->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_FINALIZE_UPDATED_RUNNING_MEAN_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &next_running_mean_desc)); + + // Set NEXT_RUNNING_VAR output tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(NEXT_RUNNING_VAR, + BN_finalize_attributes::output_names::NEXT_RUNNING_VAR); + auto next_running_var_desc = tensors.at(NEXT_RUNNING_VAR->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_FINALIZE_UPDATED_RUNNING_VAR_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &next_running_var_desc)); + + // Set MEAN output tensor (saved mean) + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(MEAN, BN_finalize_attributes::output_names::MEAN); + auto mean_desc = tensors.at(MEAN->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_FINALIZE_SAVED_MEAN_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &mean_desc)); + + // Set INV_VARIANCE output tensor (saved inv std) + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(INV_VARIANCE, BN_finalize_attributes::output_names::INV_VARIANCE); + auto inv_var_desc = tensors.at(INV_VARIANCE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_FINALIZE_SAVED_INV_STD_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &inv_var_desc)); + + // Set EPSILON tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(EPSILON, BN_finalize_attributes::input_names::EPSILON); + auto epsilon_desc = tensors.at(EPSILON->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_FINALIZE_EPSILON_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &epsilon_desc)); + + // Set MOMENTUM tensor (exp average factor) + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(MOMENTUM, BN_finalize_attributes::input_names::MOMENTUM); + auto momentum_desc = tensors.at(MOMENTUM->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_FINALIZE_EXP_AVERATE_FACTOR_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &momentum_desc)); + + // Set ACCUM_COUNT tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(ACCUM_COUNT, BN_finalize_attributes::input_names::ACCUM_COUNT); + auto accum_count_desc = tensors.at(ACCUM_COUNT->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_finalize_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_FINALIZE_ACCUM_COUNT_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &accum_count_desc)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(bn_finalize_operation.get_raw_desc())); + + operations.push_back(std::make_shared(std::move(bn_finalize_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "BN_FINALIZE"})"_json); + } +#endif +}; + +} // namespace graph + +} // namespace cudnn_frontend \ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/concatenate.h b/deps/cudnn-frontend/include/cudnn_frontend/node/concatenate.h new file mode 100644 index 0000000..5f62589 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/concatenate.h @@ -0,0 +1,163 @@ +#pragma once + +#include "../../cudnn_frontend_Logging.h" +#include "../../cudnn_frontend_shim.h" + +#include "../graph_helpers.h" +#include "../node_interface.h" +#include + +namespace cudnn_frontend { + +namespace graph { + +class ConcatenateNode : public NodeCRTP { + public: + Concatenate_attributes attributes; + + ConcatenateNode(Concatenate_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::CONCATENATE; + } + + error_t + pre_validate_node() const override final { + getLogger() << "[cudnn_frontend] INFO: " << "Validating ConcatenateNode " << attributes.name << std::endl; + + RETURN_CUDNN_FRONTEND_ERROR_IF(!attributes.axis.has_value(), error_code_t::ATTRIBUTE_NOT_SET, "Axis not set\n"); + + auto X = attributes.inputs; + + RETURN_CUDNN_FRONTEND_ERROR_IF( + (X.size() == 0), error_code_t::INVALID_VALUE, "Input size of the concatenate node cannot be zero\n"); + + return {error_code_t::OK, ""}; + } + + error_t + infer_properties_node() override final { + getLogger() << "[cudnn_frontend] INFO: Inferring properties for ConcatenateNode " << attributes.name + << std::endl; + + attributes.fill_from_context(context); + + auto Y = attributes.outputs[Concatenate_attributes::output_names::Y]; + + // Infer dims and strides only if user did not set them + int64_t dim_sum = 0; + for (const auto& input : attributes.inputs) { + dim_sum += input->get_dim()[attributes.axis.value()]; + } + + auto X = attributes.inputs[0]; + auto dims = X->get_dim(); + dims[attributes.axis.value()] = dim_sum; + + if (Y->get_dim().empty()) { + Y->set_dim(dims); + Y->set_dim(dims); + } + + if (Y->get_stride().empty()) { + std::vector stride_order; + CHECK_CUDNN_FRONTEND_ERROR( + detail::generate_stride_order_preserving_format(X->get_stride(), dims.size(), stride_order)); + Y->set_stride(detail::generate_stride(dims, stride_order)); + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set& uids_involved_in_operations, + std::vector>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map>& tensors) const override final { + getLogger() << "[cudnn_frontend] INFO: " << "Building ConcatenateNode operations " << attributes.name + << std::endl; + auto cudnn_ver_error = error_t{error_code_t::GRAPH_NOT_SUPPORTED, "Concatenate requires cuDNN v9.7.0"}; + +#if (CUDNN_VERSION >= 90700) + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(90700, cudnn_ver_error); + CUDNN_FRONTEND_UNUSED(operations); + auto concatenate_operation = make_shared_backend_pointer(CUDNN_BACKEND_OPERATION_CONCAT_DESCRIPTOR); + + std::vector backend_x(attributes.inputs.size()); + size_t index = 0; + for (const auto& input : attributes.inputs) { + backend_x[index] = tensors[input->get_uid()]->get_desc()->get_backend_descriptor(); + index++; + } + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(concatenate_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_CONCAT_INPUT_DESCS, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + attributes.inputs.size(), + backend_x.data())); + + auto Y = attributes.outputs.find(Concatenate_attributes::output_names::Y)->second; + auto backend_y = tensors[Y->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(concatenate_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_CONCAT_OUTPUT_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_y)); + + int64_t axis = attributes.axis.value(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(concatenate_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_CONCAT_AXIS, + CUDNN_TYPE_INT64, + 1, + &axis)); + + if (attributes.in_place_index.has_value()) { + int64_t in_place_index = attributes.in_place_index.value(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(concatenate_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_CONCAT_INPLACE_INDEX, + CUDNN_TYPE_INT64, + 1, + &in_place_index)); + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(concatenate_operation->get_backend_descriptor())); + + raw_operations.push_back(concatenate_operation); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; +#else + CUDNN_FRONTEND_UNUSED(uids_involved_in_operations); + CUDNN_FRONTEND_UNUSED(operations); + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FRONTEND_UNUSED(tensors); + return cudnn_ver_error; +#endif + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "CONCATENATE"})"_json); + } +#endif +}; + +inline void +INode::concatenate(std::vector> x, + Concatenate_attributes attributes, + std::shared_ptr y) { + for (auto& element : x) { + attributes.inputs.push_back(element); + } + attributes.outputs[Concatenate_attributes::output_names::Y] = y; + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); +} + +} // namespace graph + +} // namespace cudnn_frontend diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/conv_dgrad.h b/deps/cudnn-frontend/include/cudnn_frontend/node/conv_dgrad.h new file mode 100644 index 0000000..a2e2e4e --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/conv_dgrad.h @@ -0,0 +1,205 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend::graph { + +class DgradNode : public NodeCRTP { + public: + Conv_dgrad_attributes attributes; + + DgradNode(Conv_dgrad_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::DGRAD; + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating Node Type::DGRAD " << attributes.name); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.get_pre_padding().empty(), error_code_t::ATTRIBUTE_NOT_SET, "Pre padding not set."); + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.get_post_padding().empty(), error_code_t::ATTRIBUTE_NOT_SET, "Post padding not set."); + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.get_stride().empty(), error_code_t::ATTRIBUTE_NOT_SET, "Conv strides not set."); + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.get_dilation().empty(), error_code_t::ATTRIBUTE_NOT_SET, "Conv dilation not set."); + + return {error_code_t::OK, ""}; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for dgrad node " << attributes.name); + + attributes.fill_from_context(context); + + // TODO: Only inferrencing from (X, DY) -> DW works today. + auto DX = attributes.outputs.find(Conv_dgrad_attributes::output_names::DX)->second; + auto W = attributes.inputs.find(Conv_dgrad_attributes::input_names::W)->second; + auto DY = attributes.inputs.find(Conv_dgrad_attributes::input_names::DY)->second; + + auto const w_tensor_dim = W->get_dim(); + auto const dy_tensor_dim = DY->get_dim(); + auto dx_tensor_dim = DX->get_dim(); + + RETURN_CUDNN_FRONTEND_ERROR_IF(DX->get_dim().empty(), + error_code_t::ATTRIBUTE_NOT_SET, + "For dgrad node, output dimension inferencing is not possible."); + + // No dim inferencing as inverse mapping from DY, W to DX is not unique. + // Only infer strides if user did not set them + if (DX->get_stride().empty()) { + auto const& DX_dim = DX->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(DX_dim.size()); + DX->set_stride(detail::generate_stride(DX_dim, stride_order)); + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set& uids_involved_in_operations, + std::vector>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: Building DgradNode operations " << attributes.name << " "); + + // Create dgrad descriptor by directly calling cuDNN backend API + ConvDesc_v8 dgrad_descriptor; + int64_t const spatial_dim_count = attributes.get_pre_padding().size(); + + _CUDNN_CHECK_CUDNN_ERROR( + dgrad_descriptor.initialize_managed_backend_pointer(CUDNN_BACKEND_CONVOLUTION_DESCRIPTOR)); + + cudnnDataType_t cudnn_data_type; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.compute_data_type, cudnn_data_type)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dgrad_descriptor.get_raw_desc(), + CUDNN_ATTR_CONVOLUTION_COMP_TYPE, + CUDNN_TYPE_DATA_TYPE, + 1, + &cudnn_data_type)); + + cudnnConvolutionMode_t mode = detail::convert_to_cudnn_type(attributes.math_mode); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute( + dgrad_descriptor.get_raw_desc(), CUDNN_ATTR_CONVOLUTION_CONV_MODE, CUDNN_TYPE_CONVOLUTION_MODE, 1, &mode)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dgrad_descriptor.get_raw_desc(), + CUDNN_ATTR_CONVOLUTION_SPATIAL_DIMS, + CUDNN_TYPE_INT64, + 1, + &spatial_dim_count)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dgrad_descriptor.get_raw_desc(), + CUDNN_ATTR_CONVOLUTION_PRE_PADDINGS, + CUDNN_TYPE_INT64, + spatial_dim_count, + attributes.get_pre_padding().data())); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dgrad_descriptor.get_raw_desc(), + CUDNN_ATTR_CONVOLUTION_POST_PADDINGS, + CUDNN_TYPE_INT64, + spatial_dim_count, + attributes.get_post_padding().data())); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dgrad_descriptor.get_raw_desc(), + CUDNN_ATTR_CONVOLUTION_DILATIONS, + CUDNN_TYPE_INT64, + spatial_dim_count, + attributes.get_dilation().data())); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dgrad_descriptor.get_raw_desc(), + CUDNN_ATTR_CONVOLUTION_FILTER_STRIDES, + CUDNN_TYPE_INT64, + spatial_dim_count, + attributes.get_stride().data())); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(dgrad_descriptor.get_raw_desc())); + CUDNN_FE_LOG_LABEL_ENDL(dgrad_descriptor); + + // Create operation by directly calling cuDNN backend API + Operation_v8 dgrad_operation; + + _CUDNN_CHECK_CUDNN_ERROR(dgrad_operation.initialize_managed_backend_pointer( + CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_DATA_DESCRIPTOR)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DX, Conv_dgrad_attributes::output_names::DX); + auto dx_desc = tensors.at(DX->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dgrad_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_DATA_DX, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dx_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(W, Conv_dgrad_attributes::input_names::W); + auto w_desc = tensors.at(W->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dgrad_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_DATA_W, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &w_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(DY, Conv_dgrad_attributes::input_names::DY); + auto dy_desc = tensors.at(DY->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dgrad_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_DATA_DY, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dy_desc)); + + auto conv_desc_ptr = dgrad_descriptor.get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dgrad_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_DATA_CONV_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &conv_desc_ptr)); + + float alpha = 1.0f; + float beta = 0.0f; + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dgrad_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_DATA_ALPHA, + CUDNN_TYPE_FLOAT, + 1, + &alpha)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dgrad_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_DATA_BETA, + CUDNN_TYPE_FLOAT, + 1, + &beta)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(dgrad_operation.get_raw_desc())); + + operations.push_back(std::make_shared(std::move(dgrad_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "CONV_DGRAD"})"_json); + } +#endif +}; + +} // namespace cudnn_frontend::graph \ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/conv_fprop.h b/deps/cudnn-frontend/include/cudnn_frontend/node/conv_fprop.h new file mode 100644 index 0000000..34acf55 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/conv_fprop.h @@ -0,0 +1,254 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend::graph { +class ConvolutionNode : public NodeCRTP { + public: + Conv_fprop_attributes attributes; + + ConvolutionNode(Conv_fprop_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::CONVOLUTION; + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating Node Type::CONVOLUTION " << attributes.name); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.get_pre_padding().empty(), error_code_t::ATTRIBUTE_NOT_SET, "Pre padding not set."); + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.get_post_padding().empty(), error_code_t::ATTRIBUTE_NOT_SET, "Post padding not set."); + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.get_stride().empty(), error_code_t::ATTRIBUTE_NOT_SET, "Conv strides not set."); + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.get_dilation().empty(), error_code_t::ATTRIBUTE_NOT_SET, "Conv dilation not set."); + + // Implicit GEMM kernels compute B_offset as int32. When the product of all filter + // dimensions (K * C * spatial...) exceeds INT32_MAX the offset overflows, causing IMA or + // silent NaN. Conservative upper bound — exact check requires kernel tiling params not + // available at graph-validate time. Applies to 2D (4D filter) and 3D (5D filter) fprop. + auto W_it = attributes.inputs.find(Conv_fprop_attributes::input_names::W); + if (W_it != attributes.inputs.end() && W_it->second) { + auto const& w_dim = W_it->second->get_dim(); + if (w_dim.size() == 4 || w_dim.size() == 5) { + int64_t filter_elements = 1; + for (auto d : w_dim) filter_elements *= d; + RETURN_CUDNN_FRONTEND_ERROR_IF( + filter_elements > static_cast(std::numeric_limits::max()), + error_code_t::GRAPH_NOT_SUPPORTED, + "Conv filter total elements exceed INT32_MAX."); + } + } + + return {error_code_t::OK, ""}; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for conv node " << attributes.name); + + attributes.fill_from_context(context); + + // TODO: Only inferrencing from (X, W) -> Y works today. + auto& X = attributes.inputs.find(Conv_fprop_attributes::input_names::X)->second; + auto& W = attributes.inputs.find(Conv_fprop_attributes::input_names::W)->second; + auto& Y = attributes.outputs.find(Conv_fprop_attributes::output_names::Y)->second; + + auto const x_tensor_dim = X->get_dim(); + auto const w_tensor_dim = W->get_dim(); + auto y_tensor_dim = Y->get_dim(); + + // Only infer dims and strides if user did not set them + if (y_tensor_dim.empty()) { + y_tensor_dim.resize(x_tensor_dim.size()); + auto const& pre_padding = attributes.get_pre_padding(); + auto const& post_padding = attributes.get_post_padding(); + auto const& stride = attributes.get_stride(); + auto const& dilation = attributes.get_dilation(); + // N + y_tensor_dim[0] = x_tensor_dim[0]; + // PQ + for (size_t dim = 2; dim < x_tensor_dim.size(); ++dim) { + y_tensor_dim[dim] = 1 + (x_tensor_dim[dim] - dilation[dim - 2] * (w_tensor_dim[dim] - 1) - 1 + + pre_padding[dim - 2] + post_padding[dim - 2]) / + stride[dim - 2]; + } + // K + y_tensor_dim[1] = w_tensor_dim[0]; + Y->set_dim(y_tensor_dim); + } + if (Y->get_stride().empty()) { + auto const& Y_dim = Y->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(Y_dim.size()); + Y->set_stride(detail::generate_stride(Y_dim, stride_order)); + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set& uids_involved_in_operations, + std::vector>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: Building ConvolutionNode operations " << attributes.name << " "); + + // Create convolution descriptor by directly calling cuDNN backend API + ConvDesc_v8 convolution_descriptor; + int64_t const spatial_dim_count = attributes.get_pre_padding().size(); + + _CUDNN_CHECK_CUDNN_ERROR( + convolution_descriptor.initialize_managed_backend_pointer(CUDNN_BACKEND_CONVOLUTION_DESCRIPTOR)); + + // Set compute type + cudnnDataType_t cudnn_data_type; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.compute_data_type, cudnn_data_type)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_descriptor.get_raw_desc(), + CUDNN_ATTR_CONVOLUTION_COMP_TYPE, + CUDNN_TYPE_DATA_TYPE, + 1, + &cudnn_data_type)); + + // Set convolution mode + cudnnConvolutionMode_t mode = detail::convert_to_cudnn_type(attributes.math_mode); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_descriptor.get_raw_desc(), + CUDNN_ATTR_CONVOLUTION_CONV_MODE, + CUDNN_TYPE_CONVOLUTION_MODE, + 1, + &mode)); + + // Set spatial dimensions + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_descriptor.get_raw_desc(), + CUDNN_ATTR_CONVOLUTION_SPATIAL_DIMS, + CUDNN_TYPE_INT64, + 1, + &spatial_dim_count)); + + // Set pre-padding + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_descriptor.get_raw_desc(), + CUDNN_ATTR_CONVOLUTION_PRE_PADDINGS, + CUDNN_TYPE_INT64, + spatial_dim_count, + attributes.get_pre_padding().data())); + + // Set post-padding + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_descriptor.get_raw_desc(), + CUDNN_ATTR_CONVOLUTION_POST_PADDINGS, + CUDNN_TYPE_INT64, + spatial_dim_count, + attributes.get_post_padding().data())); + + // Set dilation + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_descriptor.get_raw_desc(), + CUDNN_ATTR_CONVOLUTION_DILATIONS, + CUDNN_TYPE_INT64, + spatial_dim_count, + attributes.get_dilation().data())); + + // Set strides + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_descriptor.get_raw_desc(), + CUDNN_ATTR_CONVOLUTION_FILTER_STRIDES, + CUDNN_TYPE_INT64, + spatial_dim_count, + attributes.get_stride().data())); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(convolution_descriptor.get_raw_desc())); + CUDNN_FE_LOG_LABEL_ENDL(convolution_descriptor); + + // Create operation by directly calling cuDNN backend API + Operation_v8 convolution_operation; + + _CUDNN_CHECK_CUDNN_ERROR(convolution_operation.initialize_managed_backend_pointer( + CUDNN_BACKEND_OPERATION_CONVOLUTION_FORWARD_DESCRIPTOR)); + + // Set input tensor X + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Conv_fprop_attributes::input_names::X); + auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_X, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &x_desc)); + + // Set weight tensor W + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(W, Conv_fprop_attributes::input_names::W); + auto w_desc = tensors.at(W->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_W, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &w_desc)); + + // Set output tensor Y + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(Y, Conv_fprop_attributes::output_names::Y); + auto y_desc = tensors.at(Y->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_Y, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &y_desc)); + + // Set convolution descriptor + auto conv_desc_ptr = convolution_descriptor.get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_CONV_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &conv_desc_ptr)); + + // Set alpha and beta + float alpha = 1.0f; + float beta = 0.0f; + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_ALPHA, + CUDNN_TYPE_FLOAT, + 1, + &alpha)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(convolution_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_CONVOLUTION_FORWARD_BETA, + CUDNN_TYPE_FLOAT, + 1, + &beta)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(convolution_operation.get_raw_desc())); + + operations.push_back(std::make_shared(std::move(convolution_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"({"tag": "CONV_FPROP"})"_json); + } +#endif +}; + +} // namespace cudnn_frontend::graph \ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/conv_wgrad.h b/deps/cudnn-frontend/include/cudnn_frontend/node/conv_wgrad.h new file mode 100644 index 0000000..2f9b478 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/conv_wgrad.h @@ -0,0 +1,201 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend::graph { + +class WgradNode : public NodeCRTP { + public: + Conv_wgrad_attributes attributes; + + WgradNode(Conv_wgrad_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::WGRAD; + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating Node Type::WGRAD " << attributes.name); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.get_pre_padding().empty(), error_code_t::ATTRIBUTE_NOT_SET, "Pre padding not set."); + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.get_post_padding().empty(), error_code_t::ATTRIBUTE_NOT_SET, "Post padding not set."); + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.get_stride().empty(), error_code_t::ATTRIBUTE_NOT_SET, "Conv strides not set."); + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.get_dilation().empty(), error_code_t::ATTRIBUTE_NOT_SET, "Conv dilation not set."); + + return {error_code_t::OK, ""}; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for conv node " << attributes.name); + + attributes.fill_from_context(context); + + // TODO: Only inferrencing from (X, DY) -> DW works today. + auto X = attributes.inputs[Conv_wgrad_attributes::input_names::X]; + auto DW = attributes.outputs[Conv_wgrad_attributes::output_names::DW]; + auto DY = attributes.inputs[Conv_wgrad_attributes::input_names::DY]; + + auto const x_tensor_dim = X->get_dim(); + auto const dy_tensor_dim = DY->get_dim(); + auto dw_tensor_dim = DW->get_dim(); + + // No dim inferencing as inverse mapping from DY, X to DX is not unique. + // Only infer strides if user did not set them + if (DW->get_stride().empty()) { + auto const& DW_dim = DW->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(DW_dim.size()); + DW->set_stride(detail::generate_stride(DW_dim, stride_order)); + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set& uids_involved_in_operations, + std::vector>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: Building WgradNode operations " << attributes.name << " "); + + // Create wgrad descriptor by directly calling cuDNN backend API + ConvDesc_v8 wgrad_descriptor; + int64_t const spatial_dim_count = attributes.get_pre_padding().size(); + + _CUDNN_CHECK_CUDNN_ERROR( + wgrad_descriptor.initialize_managed_backend_pointer(CUDNN_BACKEND_CONVOLUTION_DESCRIPTOR)); + + cudnnDataType_t cudnn_data_type; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.compute_data_type, cudnn_data_type)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(wgrad_descriptor.get_raw_desc(), + CUDNN_ATTR_CONVOLUTION_COMP_TYPE, + CUDNN_TYPE_DATA_TYPE, + 1, + &cudnn_data_type)); + + cudnnConvolutionMode_t mode = detail::convert_to_cudnn_type(attributes.math_mode); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute( + wgrad_descriptor.get_raw_desc(), CUDNN_ATTR_CONVOLUTION_CONV_MODE, CUDNN_TYPE_CONVOLUTION_MODE, 1, &mode)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(wgrad_descriptor.get_raw_desc(), + CUDNN_ATTR_CONVOLUTION_SPATIAL_DIMS, + CUDNN_TYPE_INT64, + 1, + &spatial_dim_count)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(wgrad_descriptor.get_raw_desc(), + CUDNN_ATTR_CONVOLUTION_PRE_PADDINGS, + CUDNN_TYPE_INT64, + spatial_dim_count, + attributes.get_pre_padding().data())); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(wgrad_descriptor.get_raw_desc(), + CUDNN_ATTR_CONVOLUTION_POST_PADDINGS, + CUDNN_TYPE_INT64, + spatial_dim_count, + attributes.get_post_padding().data())); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(wgrad_descriptor.get_raw_desc(), + CUDNN_ATTR_CONVOLUTION_DILATIONS, + CUDNN_TYPE_INT64, + spatial_dim_count, + attributes.get_dilation().data())); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(wgrad_descriptor.get_raw_desc(), + CUDNN_ATTR_CONVOLUTION_FILTER_STRIDES, + CUDNN_TYPE_INT64, + spatial_dim_count, + attributes.get_stride().data())); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(wgrad_descriptor.get_raw_desc())); + CUDNN_FE_LOG_LABEL_ENDL(wgrad_descriptor); + + // Create operation by directly calling cuDNN backend API + Operation_v8 wgrad_operation; + + _CUDNN_CHECK_CUDNN_ERROR(wgrad_operation.initialize_managed_backend_pointer( + CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_FILTER_DESCRIPTOR)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Conv_wgrad_attributes::input_names::X); + auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(wgrad_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_FILTER_X, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &x_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(DY, Conv_wgrad_attributes::input_names::DY); + auto dy_desc = tensors.at(DY->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(wgrad_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_FILTER_DY, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dy_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DW, Conv_wgrad_attributes::output_names::DW); + auto dw_desc = tensors.at(DW->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(wgrad_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_FILTER_DW, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dw_desc)); + + auto conv_desc_ptr = wgrad_descriptor.get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(wgrad_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_FILTER_CONV_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &conv_desc_ptr)); + + float alpha = 1.0f; + float beta = 0.0f; + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(wgrad_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_FILTER_ALPHA, + CUDNN_TYPE_FLOAT, + 1, + &alpha)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(wgrad_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_CONVOLUTION_BWD_FILTER_BETA, + CUDNN_TYPE_FLOAT, + 1, + &beta)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(wgrad_operation.get_raw_desc())); + + operations.push_back(std::make_shared(std::move(wgrad_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "CONV_WGRAD"})"_json); + } +#endif +}; + +} // namespace cudnn_frontend::graph \ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/dbn.h b/deps/cudnn-frontend/include/cudnn_frontend/node/dbn.h new file mode 100644 index 0000000..9f2a23e --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/dbn.h @@ -0,0 +1,206 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend { + +namespace graph { + +class DBNNode : public NodeCRTP { + public: + Batchnorm_backward_attributes attributes; + + DBNNode(Batchnorm_backward_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::DBN; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for DBN node " << attributes.name); + + attributes.fill_from_context(context); + + // TODO: Only inferencing from X works today. + auto X = attributes.inputs[Batchnorm_backward_attributes::input_names::X]; + auto const x_tensor_dim = X->get_dim(); + + auto DX = attributes.outputs[Batchnorm_backward_attributes::output_names::DX]; + auto dx_tensor_dim = DX->get_dim(); + // Only infer dims and strides if user did not set them + if (dx_tensor_dim.empty()) { + dx_tensor_dim.resize(x_tensor_dim.size()); + DX->set_dim(x_tensor_dim); + } + if (DX->get_stride().empty()) { + auto const& DX_dim = DX->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(DX_dim.size()); + DX->set_stride(detail::generate_stride(DX_dim, stride_order)); + } + + // Set channel length tensors + auto infer_per_channel_tensors = [&x_tensor_dim](std::shared_ptr& T) { + auto tensor_dim = T->get_dim(); + // Only infer dims and strides if user did not set them + if (tensor_dim.empty()) { + tensor_dim.resize(x_tensor_dim.size(), 1); + tensor_dim[1] = x_tensor_dim[1]; + T->set_dim(tensor_dim); + } + if (T->get_stride().empty()) { + auto const& T_dim = T->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(T_dim.size()); + T->set_stride(detail::generate_stride(T_dim, stride_order)); + } + }; + infer_per_channel_tensors(attributes.outputs[Batchnorm_backward_attributes::output_names::DSCALE]); + infer_per_channel_tensors(attributes.outputs[Batchnorm_backward_attributes::output_names::DBIAS]); + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set& uids_involved_in_operations, + std::vector>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: " << "Building DBNNode operations " << attributes.name << " "); + + // Create operation by directly calling cuDNN backend API + Operation_v8 dbn_operation; + + _CUDNN_CHECK_CUDNN_ERROR( + dbn_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_NORM_BACKWARD_DESCRIPTOR)); + + // Set norm mode to BATCH_NORM + cudnnBackendNormMode_t cudnn_norm_mode; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormMode_t::BATCH_NORM, cudnn_norm_mode)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dbn_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_MODE, + CUDNN_TYPE_NORM_MODE, + 1, + &cudnn_norm_mode)); + + // Set input tensor X + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Batchnorm_backward_attributes::input_names::X); + auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dbn_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &x_desc)); + + // Set DY tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(DY, Batchnorm_backward_attributes::input_names::DY); + auto dy_desc = tensors.at(DY->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dbn_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_DYDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dy_desc)); + + // Set scale tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SCALE, Batchnorm_backward_attributes::input_names::SCALE); + auto scale_desc = tensors.at(SCALE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dbn_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_SCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &scale_desc)); + + // Set mean and inv_variance tensors + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(MEAN, Batchnorm_backward_attributes::input_names::MEAN); + auto mean_desc = tensors.at(MEAN->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dbn_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_MEAN_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &mean_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(INV_VARIANCE, + Batchnorm_backward_attributes::input_names::INV_VARIANCE); + auto inv_var_desc = tensors.at(INV_VARIANCE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dbn_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_INV_VARIANCE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &inv_var_desc)); + + // Set DSCALE and DBIAS output tensors + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DSCALE, Batchnorm_backward_attributes::output_names::DSCALE); + auto dscale_desc = tensors.at(DSCALE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dbn_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_DSCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dscale_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DBIAS, Batchnorm_backward_attributes::output_names::DBIAS); + auto dbias_desc = tensors.at(DBIAS->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dbn_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_DBIAS_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dbias_desc)); + + // Set DX output tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DX, Batchnorm_backward_attributes::output_names::DX); + auto dx_desc = tensors.at(DX->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dbn_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_DXDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dx_desc)); + + // Set peer stat tensors if any + if (!attributes.peer_stats.empty()) { + std::vector peer_stat_descs; + for (auto const& peer_stat : attributes.peer_stats) { + peer_stat_descs.push_back(tensors.at(peer_stat->get_uid())->get_raw_desc()); + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dbn_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_PEER_STAT_DESCS, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + peer_stat_descs.size(), + peer_stat_descs.data())); + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(dbn_operation.get_raw_desc())); + + operations.push_back(std::make_shared(std::move(dbn_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "DBN"})"_json); + } +#endif +}; + +} // namespace graph + +} // namespace cudnn_frontend \ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/dbn_weight.h b/deps/cudnn-frontend/include/cudnn_frontend/node/dbn_weight.h new file mode 100644 index 0000000..bfad0df --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/dbn_weight.h @@ -0,0 +1,215 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend { + +namespace graph { + +class DBNWeightNode : public NodeCRTP { + public: + DBN_weight_attributes attributes; + + DBNWeightNode(DBN_weight_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::DBN_WEIGHT; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for batchnorm finalize node " << attributes.name); + + attributes.fill_from_context(context); + + // TODO: Only inferencing from DY works today. + auto DY = attributes.inputs[DBN_weight_attributes::input_names::DY]; + auto const dy_tensor_dim = DY->get_dim(); + + auto X = attributes.inputs[DBN_weight_attributes::input_names::X]; + auto x_tensor_dim = X->get_dim(); + // Only infer dims and strides if user did not set them + if (x_tensor_dim.empty()) { + x_tensor_dim.resize(dy_tensor_dim.size()); + X->set_dim(dy_tensor_dim); + } + if (X->get_stride().empty()) { + auto const& X_dim = X->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(X_dim.size()); + X->set_stride(detail::generate_stride(X_dim, stride_order)); + } + + // Set channel length tensors + auto infer_per_channel_tensors = [&dy_tensor_dim](std::shared_ptr const& T) { + auto tensor_dim = T->get_dim(); + // Only infer dims and strides if user did not set them + if (T->get_dim().empty()) { + tensor_dim.resize(dy_tensor_dim.size(), 1); + tensor_dim[1] = dy_tensor_dim[1]; + T->set_dim(tensor_dim); + } + if (T->get_stride().empty()) { + auto const& T_dim = T->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(T_dim.size()); + T->set_stride(detail::generate_stride(T_dim, stride_order)); + } + }; + infer_per_channel_tensors(attributes.outputs[DBN_weight_attributes::output_names::DBIAS]); + infer_per_channel_tensors(attributes.outputs[DBN_weight_attributes::output_names::DSCALE]); + infer_per_channel_tensors(attributes.outputs[DBN_weight_attributes::output_names::EQ_BIAS]); + infer_per_channel_tensors(attributes.outputs[DBN_weight_attributes::output_names::EQ_SCALE_DY]); + infer_per_channel_tensors(attributes.outputs[DBN_weight_attributes::output_names::EQ_SCALE_X]); + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set& uids_involved_in_operations, + std::vector>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: " << "Building DBNWeightNode operations " << attributes.name << " "); + + // Create operation by directly calling cuDNN backend API + Operation_v8 bn_bwd_weight_operation; + + _CUDNN_CHECK_CUDNN_ERROR(bn_bwd_weight_operation.initialize_managed_backend_pointer( + CUDNN_BACKEND_OPERATION_BN_BWD_WEIGHTS_DESCRIPTOR)); + + // Set compute type (math precision) + cudnnDataType_t compute_type = CUDNN_DATA_FLOAT; + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_bwd_weight_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_MATH_PREC, + CUDNN_TYPE_DATA_TYPE, + 1, + &compute_type)); + + // Set input tensor X + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, DBN_weight_attributes::input_names::X); + auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_bwd_weight_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_X_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &x_desc)); + + // Set DY tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(DY, DBN_weight_attributes::input_names::DY); + auto dy_desc = tensors.at(DY->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_bwd_weight_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_DY_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dy_desc)); + + // Set mean tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(MEAN, DBN_weight_attributes::input_names::MEAN); + auto mean_desc = tensors.at(MEAN->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_bwd_weight_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_MEAN_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &mean_desc)); + + // Set inv_variance tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(INV_VARIANCE, DBN_weight_attributes::input_names::INV_VARIANCE); + auto inv_var_desc = tensors.at(INV_VARIANCE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_bwd_weight_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_INVSTD_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &inv_var_desc)); + + // Set scale tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SCALE, DBN_weight_attributes::input_names::SCALE); + auto scale_desc = tensors.at(SCALE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_bwd_weight_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_BN_SCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &scale_desc)); + + // Set DSCALE output tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DSCALE, DBN_weight_attributes::output_names::DSCALE); + auto dscale_desc = tensors.at(DSCALE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_bwd_weight_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_DBN_SCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dscale_desc)); + + // Set DBIAS output tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DBIAS, DBN_weight_attributes::output_names::DBIAS); + auto dbias_desc = tensors.at(DBIAS->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_bwd_weight_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_DBN_BIAS_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dbias_desc)); + + // Set EQ_SCALE_DY output tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(EQ_SCALE_DY, DBN_weight_attributes::output_names::EQ_SCALE_DY); + auto eq_scale_dy_desc = tensors.at(EQ_SCALE_DY->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_bwd_weight_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_EQ_DY_SCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &eq_scale_dy_desc)); + + // Set EQ_SCALE_X output tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(EQ_SCALE_X, DBN_weight_attributes::output_names::EQ_SCALE_X); + auto eq_scale_x_desc = tensors.at(EQ_SCALE_X->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_bwd_weight_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_EQ_X_SCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &eq_scale_x_desc)); + + // Set EQ_BIAS output tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(EQ_BIAS, DBN_weight_attributes::output_names::EQ_BIAS); + auto eq_bias_desc = tensors.at(EQ_BIAS->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(bn_bwd_weight_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_BN_BWD_WEIGHTS_EQ_BIAS, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &eq_bias_desc)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(bn_bwd_weight_operation.get_raw_desc())); + + operations.push_back(std::make_shared(std::move(bn_bwd_weight_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "DBN_WEIGHT"})"_json); + } +#endif +}; + +} // namespace graph + +} // namespace cudnn_frontend \ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/diagonal_band_mask.h b/deps/cudnn-frontend/include/cudnn_frontend/node/diagonal_band_mask.h new file mode 100644 index 0000000..243d1e3 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/diagonal_band_mask.h @@ -0,0 +1,474 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +#include "pointwise.h" +#include "reduction.h" + +namespace cudnn_frontend::graph { + +// Base class for nodes that represent a diagonal band mask operation +// (which is either a left-bound or a right-bound mask, but not both at the same time). +template +class DiagonalBandMaskNodeBase : public NodeCRTP { + protected: + using input_names = DiagonalBandMask_attributes::input_names; + using output_names = DiagonalBandMask_attributes::output_names; + + public: + DiagonalBandMask_attributes attributes; + + DiagonalBandMaskNodeBase(DiagonalBandMask_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating DiagonalBandMaskNode " << attributes.name); + + RETURN_CUDNN_FRONTEND_ERROR_IF(has_left_bound() && has_shift_right_bound(), + error_code_t::INVALID_VALUE, + "DiagonalBandMaskNode cannot have both left_bound and shift_right_bound"); + RETURN_CUDNN_FRONTEND_ERROR_IF( + (has_seq_len_q() || has_seq_len_kv()) && (has_cu_seq_len_q() || has_cu_seq_len_kv()), + error_code_t::INVALID_VALUE, + "SEQ_LEN_Q / SEQ_LEN_KV and CU_SEQ_LEN_Q / CU_SEQ_LEN_KV are mutually exclusive"); + + return pre_validate_node_extra(); + } + + virtual error_t + pre_validate_node_extra() const { + return {error_code_t::OK, ""}; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for DiagonalBandMask node " << attributes.name); + + attributes.fill_from_context(this->context); + + auto input_tensor = attributes.inputs.at(DiagonalBandMask_attributes::input_names::X); + auto output_tensor = attributes.outputs.at(DiagonalBandMask_attributes::output_names::Y); + + // Only infer dims and strides if user did not set them + if (output_tensor->get_dim().empty()) { + output_tensor->set_dim(input_tensor->get_dim()); + } + + if (output_tensor->get_stride().empty()) { + output_tensor->set_stride(input_tensor->get_stride()); + } + + return {error_code_t::OK, ""}; + } + + protected: + bool + has_seq_len_q() const { + auto seq_len_Q_it = attributes.inputs.find(DiagonalBandMask_attributes::input_names::SEQ_LEN_Q); + return ((seq_len_Q_it) != attributes.inputs.end() && seq_len_Q_it->second != nullptr); + } + + bool + has_seq_len_kv() const { + auto seq_len_KV_it = attributes.inputs.find(DiagonalBandMask_attributes::input_names::SEQ_LEN_KV); + return ((seq_len_KV_it) != attributes.inputs.end() && seq_len_KV_it->second != nullptr); + } + + bool + has_cu_seq_len_q() const { + auto cu_seq_len_Q_it = attributes.inputs.find(DiagonalBandMask_attributes::input_names::CU_SEQ_LEN_Q); + return ((cu_seq_len_Q_it) != attributes.inputs.end() && cu_seq_len_Q_it->second != nullptr); + } + + bool + has_cu_seq_len_kv() const { + auto cu_seq_len_KV_it = attributes.inputs.find(DiagonalBandMask_attributes::input_names::CU_SEQ_LEN_KV); + return ((cu_seq_len_KV_it) != attributes.inputs.end() && cu_seq_len_KV_it->second != nullptr); + } + + bool + has_left_bound() const { + auto left_bound_it = attributes.inputs.find(DiagonalBandMask_attributes::input_names::LeftBound); + return ((left_bound_it) != attributes.inputs.end() && left_bound_it->second != nullptr); + } + + bool + has_right_bound() const { + return !has_left_bound(); + } + + bool + has_shift_right_bound() const { + auto shift_right_bound_it = attributes.inputs.find(DiagonalBandMask_attributes::input_names::ShiftRightBound); + return ((shift_right_bound_it) != attributes.inputs.end() && shift_right_bound_it->second != nullptr); + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + } +#endif +}; + +// Fallback implementation of diagonal band mask node that represents the mask as a series of pointwise operations. +// This is used for cuDNN versions before v9.21.0. +// Note: the individual masks can be constructed in different ways as well that yield functionally +// correct results. However, for performance reasons in the cuDNN backend they are organized as they are. Be +// cautious of performance when editing. +class CompositeDiagonalBandMaskNode : public DiagonalBandMaskNodeBase { + public: + CompositeDiagonalBandMaskNode(DiagonalBandMask_attributes&& attributes_, detail::Context const& context) + : DiagonalBandMaskNodeBase(std::move(attributes_), context) {} + + Type + getType() override final { + return Type::COMPOSITE; + } + + error_t + pre_validate_node_extra() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating CompositeDiagonalBandMaskNode " << attributes.name); + + RETURN_CUDNN_FRONTEND_ERROR_IF(has_cu_seq_len_q() || has_cu_seq_len_kv(), + error_code_t::GRAPH_NOT_SUPPORTED, + "CompositeDiagonalBandMaskNode does not support CU_SEQ_LEN_Q or CU_SEQ_LEN_KV"); + + return {error_code_t::OK, ""}; + } + + error_t + expand_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for CompositeDiagonalBandMaskNode " + << attributes.name); + + attributes.fill_from_context(context); + + auto attention_score = attributes.inputs[DiagonalBandMask_attributes::input_names::X]; + auto negative_inf_value = attributes.inputs[DiagonalBandMask_attributes::input_names::B]; + + auto row_index = pointwise( + attention_score, + Pointwise_attributes().set_name("gen_row_index").set_mode(PointwiseMode_t::GEN_INDEX).set_axis(2)); + row_index->set_data_type(DataType_t::INT32); + auto col_index = pointwise( + attention_score, + Pointwise_attributes().set_name("gen_col_index").set_mode(PointwiseMode_t::GEN_INDEX).set_axis(3)); + col_index->set_data_type(DataType_t::INT32); + + std::shared_ptr swa_comparison_output; + + if (has_right_bound()) { + std::string row_index_name = "row"; + + if (has_seq_len_kv()) { + auto seq_len_kv = attributes.inputs[DiagonalBandMask_attributes::input_names::SEQ_LEN_KV]; + row_index = pointwise(row_index, + seq_len_kv, + Pointwise_attributes() + .set_name(row_index_name += "+skv") + .set_mode(PointwiseMode_t::ADD) + .set_compute_data_type(DataType_t::INT32) + .set_axis(3)); + row_index->set_data_type(DataType_t::INT32); + } + + if (has_seq_len_q()) { + auto seq_len_q = attributes.inputs[DiagonalBandMask_attributes::input_names::SEQ_LEN_Q]; + row_index = pointwise(row_index, + seq_len_q, + Pointwise_attributes() + .set_name(row_index_name += "-sq") + .set_mode(PointwiseMode_t::SUB) + .set_compute_data_type(DataType_t::INT32) + .set_axis(3)); + row_index->set_data_type(DataType_t::INT32); + } + + if (has_shift_right_bound()) { + auto shift_right_bound = attributes.inputs[DiagonalBandMask_attributes::input_names::ShiftRightBound]; + // Use shift_right_bound's data type + row_index = pointwise(row_index, + shift_right_bound, + Pointwise_attributes() + .set_name(row_index_name += "+window") + .set_mode(PointwiseMode_t::ADD) + .set_compute_data_type(shift_right_bound->get_data_type()) + .set_axis(3)); + row_index->set_data_type(shift_right_bound->get_data_type()); + } + + swa_comparison_output = pointwise(row_index, + col_index, + Pointwise_attributes() + .set_name(row_index_name += ">col") + .set_mode(attributes.comparison_mode) + .set_compute_data_type(DataType_t::BOOLEAN)); + swa_comparison_output->set_data_type(DataType_t::BOOLEAN); + } + + // Set the left bound + else if (has_left_bound()) { + auto left_bound = attributes.inputs[DiagonalBandMask_attributes::input_names::LeftBound]; + std::string col_index_name = "col"; + + // Use left_bound's data type + col_index = pointwise(col_index, + left_bound, + Pointwise_attributes() + .set_name(col_index_name += "+window") + .set_mode(PointwiseMode_t::ADD) + .set_compute_data_type(left_bound->get_data_type()) + .set_axis(3)); + col_index->set_data_type(left_bound->get_data_type()); + + if (has_seq_len_kv()) { + auto seq_len_kv = attributes.inputs[DiagonalBandMask_attributes::input_names::SEQ_LEN_KV]; + col_index = pointwise(col_index, + seq_len_kv, + Pointwise_attributes() + .set_name(col_index_name += "-skv") + .set_mode(PointwiseMode_t::SUB) + .set_compute_data_type(DataType_t::INT32) + .set_axis(3)); + col_index->set_data_type(DataType_t::INT32); + } + + if (has_seq_len_q()) { + auto seq_len_q = attributes.inputs[DiagonalBandMask_attributes::input_names::SEQ_LEN_Q]; + col_index = pointwise(col_index, + seq_len_q, + Pointwise_attributes() + .set_name(col_index_name += "+sq") + .set_mode(PointwiseMode_t::ADD) + .set_compute_data_type(DataType_t::INT32) + .set_axis(3)); + col_index->set_data_type(DataType_t::INT32); + } + + swa_comparison_output = pointwise(col_index, + row_index, + Pointwise_attributes() + .set_name(col_index_name += ">row") + .set_mode(attributes.comparison_mode) + .set_compute_data_type(DataType_t::BOOLEAN)); + swa_comparison_output->set_data_type(DataType_t::BOOLEAN); + } else { + // This should not be reached (since one of has_left_bound() or has_right_bound() is always true by + // construction), just here for completeness. + return {error_code_t::INVALID_VALUE, + "CompositeDiagonalBandMaskNode must have a left bound or a right bound"}; + } + + // Special non-functional-style call. Needed because output already created and provided to user. + pointwise(attention_score, + negative_inf_value, + swa_comparison_output, + Pointwise_attributes().set_name("binary_select").set_mode(PointwiseMode_t::BINARY_SELECT), + attributes.outputs[DiagonalBandMask_attributes::output_names::Y]); + + return {error_code_t::OK, ""}; + } +}; + +// Newer implementation of diagonal band mask node that represents the mask as a single operation. +// This is used for cuDNN versions v9.21.0 and later. +class UnifiedDiagonalBandMaskNode : public DiagonalBandMaskNodeBase { + public: + UnifiedDiagonalBandMaskNode(DiagonalBandMask_attributes&& attributes_, detail::Context const& context) + : DiagonalBandMaskNodeBase(std::move(attributes_), context) {} + + Type + getType() override final { + return Type::DIAGONAL_BAND_MASK; + } + + error_t + create_cudnn_operations( + std::unordered_set& uids_involved_in_operations, + std::vector>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map>& tensors) const override final { + getLogger() << "[cudnn_frontend] INFO: " << "Building UnifiedDiagonalBandMaskNode operations " + << attributes.name << std::endl; + auto cudnn_ver_error = + error_t{error_code_t::GRAPH_NOT_SUPPORTED, "UnifiedDiagonalBandMaskNode requires cuDNN v9.21.0"}; + +#if (CUDNN_VERSION >= 92100) + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(92100, cudnn_ver_error); + CUDNN_FRONTEND_UNUSED(operations); + auto diagonal_band_mask_operation = make_shared_backend_pointer( + (cudnnBackendDescriptorType_t)CUDNN_BACKEND_OPERATION_DIAGONAL_BAND_MASK_DESCRIPTOR); + + auto X = attributes.inputs.find(DiagonalBandMask_attributes::input_names::X)->second; + auto backend_x = tensors[X->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(diagonal_band_mask_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_x)); + + auto B = attributes.inputs.find(DiagonalBandMask_attributes::input_names::B)->second; + auto backend_b = tensors[B->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(diagonal_band_mask_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_BDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_b)); + + if (has_seq_len_q()) { + auto seq_len_Q = attributes.inputs.find(DiagonalBandMask_attributes::input_names::SEQ_LEN_Q)->second; + auto backend_seq_len_Q = tensors[seq_len_Q->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(diagonal_band_mask_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_SEQ_LEN_QDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_seq_len_Q)); + } + + if (has_seq_len_kv()) { + auto seq_len_KV = attributes.inputs.find(DiagonalBandMask_attributes::input_names::SEQ_LEN_KV)->second; + auto backend_seq_len_KV = tensors[seq_len_KV->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(diagonal_band_mask_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_SEQ_LEN_KVDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_seq_len_KV)); + } + + if (has_cu_seq_len_q() || has_cu_seq_len_kv()) { + auto cu_seq_len_cudnn_ver_error = + error_t{error_code_t::GRAPH_NOT_SUPPORTED, + "Cumulative sequence length tensors in unified DiagonalBandMask node require cuDNN 9.24.0"}; +#if (CUDNN_VERSION >= 92400) + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(92400, cu_seq_len_cudnn_ver_error); + + if (has_cu_seq_len_q()) { + auto cu_seq_len_Q = + attributes.inputs.find(DiagonalBandMask_attributes::input_names::CU_SEQ_LEN_Q)->second; + auto backend_cu_seq_len_Q = tensors[cu_seq_len_Q->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(diagonal_band_mask_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_CU_SEQ_LEN_QDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_cu_seq_len_Q)); + } + + if (has_cu_seq_len_kv()) { + auto cu_seq_len_KV = + attributes.inputs.find(DiagonalBandMask_attributes::input_names::CU_SEQ_LEN_KV)->second; + auto backend_cu_seq_len_KV = tensors[cu_seq_len_KV->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR( + detail::set_attribute(diagonal_band_mask_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_CU_SEQ_LEN_KVDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_cu_seq_len_KV)); + } +#else + return cu_seq_len_cudnn_ver_error; +#endif + } + + if (has_left_bound()) { + auto left_bound = attributes.inputs.find(DiagonalBandMask_attributes::input_names::LeftBound)->second; + auto backend_left_bound = tensors[left_bound->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(diagonal_band_mask_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_LEFT_BOUND_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_left_bound)); + } + + if (has_shift_right_bound()) { + auto shift_right_bound = + attributes.inputs.find(DiagonalBandMask_attributes::input_names::ShiftRightBound)->second; + auto backend_shift_right_bound = + tensors[shift_right_bound->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR( + detail::set_attribute(diagonal_band_mask_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_SHIFT_RIGHT_BOUND_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_shift_right_bound)); + } + + auto Y = attributes.outputs.find(DiagonalBandMask_attributes::output_names::Y)->second; + auto backend_y = tensors[Y->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(diagonal_band_mask_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_YDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_y)); + + cudnnPointwiseMode_t cudnn_pointwise_mode; + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.comparison_mode, cudnn_pointwise_mode)); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(diagonal_band_mask_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_DIAGONAL_BAND_MASK_COMPARISON_MODE, + CUDNN_TYPE_POINTWISE_MODE, + 1, + &cudnn_pointwise_mode)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(diagonal_band_mask_operation->get_backend_descriptor())); + + raw_operations.push_back(diagonal_band_mask_operation); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; +#else + CUDNN_FRONTEND_UNUSED(uids_involved_in_operations); + CUDNN_FRONTEND_UNUSED(operations); + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FRONTEND_UNUSED(tensors); + return cudnn_ver_error; +#endif // CUDNN_VERSION >= 92100 + } +}; + +// Factory function to create a diagonal band mask node. +// This function will create the appropriate node based on the cuDNN version. +inline std::shared_ptr +INode::diagonal_band_mask(std::shared_ptr x, + std::shared_ptr b, + std::shared_ptr seq_len_q, + std::shared_ptr seq_len_kv, + std::shared_ptr cu_seq_len_q, + std::shared_ptr cu_seq_len_kv, + std::shared_ptr left_bound, + std::shared_ptr shift_right_bound, + DiagonalBandMask_attributes attributes) { + attributes.inputs[DiagonalBandMask_attributes::input_names::X] = x; + attributes.inputs[DiagonalBandMask_attributes::input_names::B] = b; + if (seq_len_q) { + attributes.inputs[DiagonalBandMask_attributes::input_names::SEQ_LEN_Q] = seq_len_q; + } + if (seq_len_kv) { + attributes.inputs[DiagonalBandMask_attributes::input_names::SEQ_LEN_KV] = seq_len_kv; + } + if (cu_seq_len_q) { + attributes.inputs[DiagonalBandMask_attributes::input_names::CU_SEQ_LEN_Q] = cu_seq_len_q; + } + if (cu_seq_len_kv) { + attributes.inputs[DiagonalBandMask_attributes::input_names::CU_SEQ_LEN_KV] = cu_seq_len_kv; + } + if (left_bound) { + attributes.inputs[DiagonalBandMask_attributes::input_names::LeftBound] = left_bound; + } + if (shift_right_bound) { + attributes.inputs[DiagonalBandMask_attributes::input_names::ShiftRightBound] = shift_right_bound; + } + auto y = attributes.outputs[DiagonalBandMask_attributes::output_names::Y] = output_tensor(attributes.name + "::Y"); + + // Newer versions of cuDNN can represent the diagonal band mask as a single operation. + if (std::min(detail::get_compiled_version(), detail::get_backend_version()) >= 92100) { + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + } else { + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + } + return y; +} + +} // namespace cudnn_frontend::graph \ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/dln.h b/deps/cudnn-frontend/include/cudnn_frontend/node/dln.h new file mode 100644 index 0000000..ad7ad1f --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/dln.h @@ -0,0 +1,226 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend { + +namespace graph { + +class DLNNode : public NodeCRTP { + public: + Layernorm_backward_attributes attributes; + + DLNNode(Layernorm_backward_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::DLN; + } + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for DLN node " << attributes.name); + + if (detail::get_backend_version() < 8906) { + attributes.inputs[Layernorm_backward_attributes::input_names::EPSILON] = + std::make_shared(0.0f); + } + + attributes.fill_from_context(context); + + // TODO: Only inferencing from X works today. + auto X = attributes.inputs[Layernorm_backward_attributes::input_names::X]; + auto const x_tensor_dim = X->get_dim(); + + auto DY = attributes.inputs[Layernorm_backward_attributes::input_names::DY]; + auto dy_tensor_dim = DY->get_dim(); + + // Only infer dims and strides if user did not set them + if (dy_tensor_dim.empty()) { + dy_tensor_dim.resize(x_tensor_dim.size()); + DY->set_dim(x_tensor_dim); + } + if (DY->get_stride().empty()) { + auto const& DY_dim = DY->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(DY_dim.size()); + DY->set_stride(detail::generate_stride(DY_dim, stride_order)); + } + + auto DX = attributes.outputs[Layernorm_backward_attributes::output_names::DX]; + auto dx_tensor_dim = DX->get_dim(); + // Only infer dims and strides if user did not set them + if (dx_tensor_dim.empty()) { + dx_tensor_dim.resize(x_tensor_dim.size()); + DX->set_dim(x_tensor_dim); + } + if (DX->get_stride().empty()) { + auto const& DX_dim = DX->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(DX_dim.size()); + DX->set_stride(detail::generate_stride(DX_dim, stride_order)); + } + + auto scale_bias_dim = X->get_dim(); + scale_bias_dim[0] = 1; + + // Set channel length tensors + auto infer_scale_bias_tensors = [&scale_bias_dim](std::shared_ptr& T) { + auto tensor_dim = T->get_dim(); + // Only infer dims and strides if user did not set them + if (tensor_dim.empty()) { + T->set_dim(scale_bias_dim); + } + if (T->get_stride().empty()) { + auto const& T_dim = T->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(T_dim.size()); + T->set_stride(detail::generate_stride(T_dim, stride_order)); + } + }; + + infer_scale_bias_tensors(attributes.outputs[Layernorm_backward_attributes::output_names::DSCALE]); + infer_scale_bias_tensors(attributes.outputs[Layernorm_backward_attributes::output_names::DBIAS]); + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set& uids_involved_in_operations, + std::vector>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: " << "Building DLNNode operations " << attributes.name << " "); + + // Create operation by directly calling cuDNN backend API + Operation_v8 dln_operation; + + _CUDNN_CHECK_CUDNN_ERROR( + dln_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_NORM_BACKWARD_DESCRIPTOR)); + + // Set norm mode to LAYER_NORM + cudnnBackendNormMode_t cudnn_norm_mode; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormMode_t::LAYER_NORM, cudnn_norm_mode)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dln_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_MODE, + CUDNN_TYPE_NORM_MODE, + 1, + &cudnn_norm_mode)); + + // Set input tensor X + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Layernorm_backward_attributes::input_names::X); + auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dln_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &x_desc)); + + // Set DY tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(DY, Layernorm_backward_attributes::input_names::DY); + auto dy_desc = tensors.at(DY->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dln_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_DYDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dy_desc)); + + // Set scale tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SCALE, Layernorm_backward_attributes::input_names::SCALE); + auto scale_desc = tensors.at(SCALE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dln_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_SCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &scale_desc)); + + // Set mean and inv_variance tensors + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(MEAN, Layernorm_backward_attributes::input_names::MEAN); + auto mean_desc = tensors.at(MEAN->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dln_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_MEAN_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &mean_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(INV_VARIANCE, + Layernorm_backward_attributes::input_names::INV_VARIANCE); + auto inv_var_desc = tensors.at(INV_VARIANCE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dln_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_INV_VARIANCE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &inv_var_desc)); + + // Set DSCALE and DBIAS output tensors + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DSCALE, Layernorm_backward_attributes::output_names::DSCALE); + auto dscale_desc = tensors.at(DSCALE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dln_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_DSCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dscale_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DBIAS, Layernorm_backward_attributes::output_names::DBIAS); + auto dbias_desc = tensors.at(DBIAS->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dln_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_DBIAS_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dbias_desc)); + + // Set DX output tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DX, Layernorm_backward_attributes::output_names::DX); + auto dx_desc = tensors.at(DX->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dln_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_DXDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dx_desc)); + + // Set epsilon tensor for older backend versions + if (detail::get_backend_version() < 8906) { + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(EPSILON, Layernorm_backward_attributes::input_names::EPSILON); + auto epsilon_desc = tensors.at(EPSILON->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(dln_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_EPSILON_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &epsilon_desc)); + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(dln_operation.get_raw_desc())); + + operations.push_back(std::make_shared(std::move(dln_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "LAYER_NORM_BPROP"})"_json); + } +#endif +}; + +} // namespace graph + +} // namespace cudnn_frontend diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/genstats.h b/deps/cudnn-frontend/include/cudnn_frontend/node/genstats.h new file mode 100644 index 0000000..8f91897 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/genstats.h @@ -0,0 +1,147 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend { + +namespace graph { + +class GenstatsNode : public NodeCRTP { + public: + Genstats_attributes attributes; + + GenstatsNode(Genstats_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::GENSTATS; + } + + error_t + infer_properties_node() override final { + attributes.fill_from_context(context); + + // Only inferrencing from X works today. + auto X = attributes.inputs[Genstats_attributes::input_names::X]; + auto SUM = attributes.outputs[Genstats_attributes::output_names::SUM]; + auto SQ_SUM = attributes.outputs[Genstats_attributes::output_names::SQ_SUM]; + + auto const x_tensor_dim = X->get_dim(); + auto sum_tensor_dim = SUM->get_dim(); + auto sq_sum_tensor_dim = SQ_SUM->get_dim(); + + // Only infer dims and strides if user did not set them + if (sum_tensor_dim.empty()) { + sum_tensor_dim.resize(x_tensor_dim.size(), 1); + sum_tensor_dim[1] = x_tensor_dim[1]; + SUM->set_dim(sum_tensor_dim); + } + if (SUM->get_stride().empty()) { + auto const& SUM_dim = SUM->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(SUM_dim.size()); + SUM->set_stride(detail::generate_stride(SUM_dim, stride_order)); + } + + // Only infer dims and strides if user did not set them + if (sq_sum_tensor_dim.empty()) { + sq_sum_tensor_dim.resize(x_tensor_dim.size(), 1); + sq_sum_tensor_dim[1] = x_tensor_dim[1]; + SQ_SUM->set_dim(sq_sum_tensor_dim); + } + if (SQ_SUM->get_stride().empty()) { + auto const& SQ_SUM_dim = SQ_SUM->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(SQ_SUM_dim.size()); + SQ_SUM->set_stride(detail::generate_stride(SQ_SUM_dim, stride_order)); + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set& uids_involved_in_operations, + std::vector>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: " << "Building GenstatsNode operations " << attributes.name << " "); + + // Create operation by directly calling cuDNN backend API + Operation_v8 genstats_operation; + + _CUDNN_CHECK_CUDNN_ERROR( + genstats_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_GEN_STATS_DESCRIPTOR)); + + // Set input tensor X + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Genstats_attributes::input_names::X); + auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(genstats_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_GENSTATS_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &x_desc)); + + // Set gen stats mode + cudnnGenStatsMode_t genstats_mode = CUDNN_GENSTATS_SUM_SQSUM; + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(genstats_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_GENSTATS_MODE, + CUDNN_TYPE_GENSTATS_MODE, + 1, + &genstats_mode)); + + // Set math precision based on X tensor data type + cudnnDataType_t math_prec = static_cast(tensors.at(X->second->get_uid())->getDataType()); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(genstats_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_GENSTATS_MATH_PREC, + CUDNN_TYPE_DATA_TYPE, + 1, + &math_prec)); + + // Set SUM output tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(SUM, Genstats_attributes::output_names::SUM); + auto sum_desc = tensors.at(SUM->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(genstats_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_GENSTATS_SUMDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &sum_desc)); + + // Set SQ_SUM output tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(SQ_SUM, Genstats_attributes::output_names::SQ_SUM); + auto sq_sum_desc = tensors.at(SQ_SUM->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(genstats_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_GENSTATS_SQSUMDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &sq_sum_desc)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(genstats_operation.get_raw_desc())); + + operations.push_back(std::make_shared(std::move(genstats_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "GENSTATS"})"_json); + } +#endif +}; + +} // namespace graph + +} // namespace cudnn_frontend \ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/instancenorm.h b/deps/cudnn-frontend/include/cudnn_frontend/node/instancenorm.h new file mode 100644 index 0000000..1b71f4a --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/instancenorm.h @@ -0,0 +1,414 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend { + +namespace graph { +class InstanceNormNode : public NodeCRTP { + public: + Instancenorm_attributes attributes; + + InstanceNormNode(Instancenorm_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::INSTANCENORM; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for instancenorm node " << attributes.name); + + attributes.fill_from_context(context); + + auto X = attributes.inputs[Instancenorm_attributes::input_names::X]; + auto Y = attributes.outputs[Instancenorm_attributes::output_names::Y]; + + // Only infer dims and strides if user did not set them + if (Y->get_dim().empty()) { + Y->set_dim(X->get_dim()); + } + if (Y->get_stride().empty()) { + Y->set_stride(X->get_stride()); + } + + // mean inv_var dim is n,c,1,1 + auto stats_dim = X->get_dim(); + for (size_t i = 2; i < stats_dim.size(); i++) { + stats_dim[i] = 1; + } + + if (attributes.forward_phase == NormFwdPhase_t::TRAINING) { + auto mean = attributes.outputs[Instancenorm_attributes::output_names::MEAN]; + if (mean->get_dim().empty()) { + mean->set_dim(stats_dim); + } + if (mean->get_stride().empty()) { + auto const& mean_dim = mean->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(mean_dim.size()); + mean->set_stride(detail::generate_stride(mean_dim, stride_order)); + } + + auto inv_var = attributes.outputs[Instancenorm_attributes::output_names::INV_VARIANCE]; + if (inv_var->get_dim().empty()) { + inv_var->set_dim(stats_dim); + } + if (inv_var->get_stride().empty()) { + auto const& inv_var_dim = inv_var->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(inv_var_dim.size()); + inv_var->set_stride(detail::generate_stride(inv_var_dim, stride_order)); + } + } + return {error_code_t::OK, ""}; + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating InstanceNormNode " << attributes.name); + + // Norm forward phase should be set + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.forward_phase == NormFwdPhase_t::NOT_SET, + error_code_t::ATTRIBUTE_NOT_SET, + "Forward phase not set of instancenorm node."); + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set& uids_involved_in_operations, + std::vector>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: Building InstanceNormNode operations " << attributes.name << " "); + + // Create operation by directly calling cuDNN backend API + Operation_v8 instancenorm_operation; + + _CUDNN_CHECK_CUDNN_ERROR( + instancenorm_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_NORM_FORWARD_DESCRIPTOR)); + + // Set norm mode to INSTANCE_NORM + cudnnBackendNormMode_t cudnn_norm_mode; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormMode_t::INSTANCE_NORM, cudnn_norm_mode)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(instancenorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_MODE, + CUDNN_TYPE_NORM_MODE, + 1, + &cudnn_norm_mode)); + + // Set forward phase + cudnnBackendNormFwdPhase_t cudnn_norm_fwd_phase; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.forward_phase, cudnn_norm_fwd_phase)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(instancenorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_PHASE, + CUDNN_TYPE_NORM_FWD_PHASE, + 1, + &cudnn_norm_fwd_phase)); + + // Set input tensor X + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Instancenorm_attributes::input_names::X); + auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(instancenorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &x_desc)); + + // Set scale and bias tensors + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SCALE, Instancenorm_attributes::input_names::SCALE); + auto scale_desc = tensors.at(SCALE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(instancenorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_SCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &scale_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(BIAS, Instancenorm_attributes::input_names::BIAS); + auto bias_desc = tensors.at(BIAS->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(instancenorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_BIAS_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &bias_desc)); + + // Set epsilon tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(EPSILON, Instancenorm_attributes::input_names::EPSILON); + auto epsilon_desc = tensors.at(EPSILON->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(instancenorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_EPSILON_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &epsilon_desc)); + + // Set output tensor Y + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(Y, Instancenorm_attributes::output_names::Y); + auto y_desc = tensors.at(Y->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(instancenorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_YDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &y_desc)); + + // Set mean and inv_variance for training phase + if (attributes.forward_phase == NormFwdPhase_t::TRAINING) { + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(MEAN, Instancenorm_attributes::output_names::MEAN); + auto mean_desc = tensors.at(MEAN->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(instancenorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_MEAN_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &mean_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(INV_VARIANCE, + Instancenorm_attributes::output_names::INV_VARIANCE); + auto inv_var_desc = tensors.at(INV_VARIANCE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(instancenorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_INV_VARIANCE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &inv_var_desc)); + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(instancenorm_operation.get_raw_desc())); + + operations.push_back(std::make_shared(std::move(instancenorm_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "INSTANCE_NORM"})"_json); + } +#endif +}; + +class DINNode : public NodeCRTP { + public: + Instancenorm_backward_attributes attributes; + + DINNode(Instancenorm_backward_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::DIN; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for DIN node " << attributes.name); + + attributes.fill_from_context(context); + + // TODO: Only inferencing from X works today. + auto X = attributes.inputs[Instancenorm_backward_attributes::input_names::X]; + auto const x_tensor_dim = X->get_dim(); + + auto DY = attributes.inputs[Instancenorm_backward_attributes::input_names::DY]; + auto dy_tensor_dim = DY->get_dim(); + + // Only infer dims and strides if user did not set them + if (dy_tensor_dim.empty()) { + dy_tensor_dim.resize(x_tensor_dim.size()); + DY->set_dim(x_tensor_dim); + } + if (DY->get_stride().empty()) { + auto const& DY_dim = DY->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(DY_dim.size()); + DY->set_stride(detail::generate_stride(DY_dim, stride_order)); + } + + auto DX = attributes.outputs[Instancenorm_backward_attributes::output_names::DX]; + auto dx_tensor_dim = DX->get_dim(); + // Only infer dims and strides if user did not set them + if (dx_tensor_dim.empty()) { + dx_tensor_dim.resize(x_tensor_dim.size()); + DX->set_dim(x_tensor_dim); + } + if (DX->get_stride().empty()) { + auto const& DX_dim = DX->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(DX_dim.size()); + DX->set_stride(detail::generate_stride(DX_dim, stride_order)); + } + + // scale_bias dim is 1,c,1,1 + // mean inv_var dim is n,c,1,1 + auto scale_bias_dim = X->get_dim(); + for (size_t i = 0; i < scale_bias_dim.size(); i++) { + if (i != 1) { + scale_bias_dim[i] = 1; + } + } + + // Set channel length tensors + auto infer_scale_bias_tensors = [&scale_bias_dim](std::shared_ptr& T) { + auto tensor_dim = T->get_dim(); + // Only infer dims and strides if user did not set them + if (tensor_dim.empty()) { + T->set_dim(scale_bias_dim); + } + if (T->get_stride().empty()) { + auto const& T_dim = T->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(T_dim.size()); + T->set_stride(detail::generate_stride(T_dim, stride_order)); + } + }; + + infer_scale_bias_tensors(attributes.outputs[Instancenorm_backward_attributes::output_names::DSCALE]); + infer_scale_bias_tensors(attributes.outputs[Instancenorm_backward_attributes::output_names::DBIAS]); + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set& uids_involved_in_operations, + std::vector>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: Building DINNode operations " << attributes.name << " "); + + // Create operation by directly calling cuDNN backend API + Operation_v8 din_operation; + + _CUDNN_CHECK_CUDNN_ERROR( + din_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_NORM_BACKWARD_DESCRIPTOR)); + + // Set norm mode to INSTANCE_NORM + cudnnBackendNormMode_t cudnn_norm_mode; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormMode_t::INSTANCE_NORM, cudnn_norm_mode)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(din_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_MODE, + CUDNN_TYPE_NORM_MODE, + 1, + &cudnn_norm_mode)); + + // Set input tensor X + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Instancenorm_backward_attributes::input_names::X); + auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(din_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &x_desc)); + + // Set DY tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(DY, Instancenorm_backward_attributes::input_names::DY); + auto dy_desc = tensors.at(DY->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(din_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_DYDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dy_desc)); + + // Set scale tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SCALE, Instancenorm_backward_attributes::input_names::SCALE); + auto scale_desc = tensors.at(SCALE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(din_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_SCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &scale_desc)); + + // Set mean and inv_variance tensors + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(MEAN, Instancenorm_backward_attributes::input_names::MEAN); + auto mean_desc = tensors.at(MEAN->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(din_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_MEAN_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &mean_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(INV_VARIANCE, + Instancenorm_backward_attributes::input_names::INV_VARIANCE); + auto inv_var_desc = tensors.at(INV_VARIANCE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(din_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_INV_VARIANCE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &inv_var_desc)); + + // Set DSCALE and DBIAS output tensors + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DSCALE, Instancenorm_backward_attributes::output_names::DSCALE); + auto dscale_desc = tensors.at(DSCALE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(din_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_DSCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dscale_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DBIAS, Instancenorm_backward_attributes::output_names::DBIAS); + auto dbias_desc = tensors.at(DBIAS->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(din_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_DBIAS_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dbias_desc)); + + // Set DX output tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DX, Instancenorm_backward_attributes::output_names::DX); + auto dx_desc = tensors.at(DX->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(din_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_DXDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dx_desc)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(din_operation.get_raw_desc())); + + operations.push_back(std::make_shared(std::move(din_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "INSTANCE_NORM_BPROP"})"_json); + } +#endif +}; + +} // namespace graph + +} // namespace cudnn_frontend \ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/layernorm.h b/deps/cudnn-frontend/include/cudnn_frontend/node/layernorm.h new file mode 100644 index 0000000..46420aa --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/layernorm.h @@ -0,0 +1,259 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend { + +namespace graph { +class LayerNormNode : public NodeCRTP { + public: + Layernorm_attributes attributes; + + LayerNormNode(Layernorm_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::LAYERNORM; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for layernorm node " << attributes.name); + + attributes.fill_from_context(context); + + auto X = attributes.inputs[Layernorm_attributes::input_names::X]; + auto Y = attributes.outputs[Layernorm_attributes::output_names::Y]; + + // Only infer dims and strides if user did not set them + if (Y->get_dim().empty()) { + Y->set_dim(X->get_dim()); + } + if (Y->get_stride().empty()) { + Y->set_stride(X->get_stride()); + } + + // scale_bias dim is 1,c,h,w + auto scale_bias_dim = X->get_dim(); + scale_bias_dim[0] = 1; + + auto scale = attributes.inputs[Layernorm_attributes::input_names::SCALE]; + // Only infer dims and strides if user did not set them + if (scale->get_dim().empty()) { + scale->set_dim(scale_bias_dim); + } + if (scale->get_stride().empty()) { + auto const& scale_dim = scale->get_dim(); + std::vector stride_order; + CHECK_CUDNN_FRONTEND_ERROR( + detail::generate_stride_order_preserving_format(X->get_stride(), scale_dim.size(), stride_order)); + scale->set_stride(detail::generate_stride(scale_dim, stride_order)); + } + + auto bias = attributes.inputs[Layernorm_attributes::input_names::BIAS]; + // Only infer dims and strides if user did not set them + if (bias->get_dim().empty()) { + bias->set_dim(scale_bias_dim); + } + if (bias->get_stride().empty()) { + auto const& bias_dim = bias->get_dim(); + std::vector stride_order; + CHECK_CUDNN_FRONTEND_ERROR( + detail::generate_stride_order_preserving_format(X->get_stride(), bias_dim.size(), stride_order)); + bias->set_stride(detail::generate_stride(bias_dim, stride_order)); + } + + if (attributes.forward_phase == NormFwdPhase_t::TRAINING) { + // stats dim is x where scale == 1 else 1 + auto stats_dim = X->get_dim(); + for (size_t i = 0; i < stats_dim.size(); i++) { + if (scale->get_dim()[i] != 1) { + stats_dim[i] = 1; + } + } + + auto mean = attributes.outputs[Layernorm_attributes::output_names::MEAN]; + // Only infer dims and strides if user did not set them + if (mean->get_dim().empty()) { + mean->set_dim(stats_dim); + } + if (mean->get_stride().empty()) { + auto const& mean_dim = mean->get_dim(); + std::vector stride_order; + CHECK_CUDNN_FRONTEND_ERROR( + detail::generate_stride_order_preserving_format(X->get_stride(), mean_dim.size(), stride_order)); + mean->set_stride(detail::generate_stride(mean_dim, stride_order)); + } + + auto inv_var = attributes.outputs[Layernorm_attributes::output_names::INV_VARIANCE]; + // Only infer dims and strides if user did not set them + if (inv_var->get_dim().empty()) { + inv_var->set_dim(stats_dim); + } + if (inv_var->get_stride().empty()) { + auto const& inv_var_dim = inv_var->get_dim(); + std::vector stride_order; + CHECK_CUDNN_FRONTEND_ERROR( + detail::generate_stride_order_preserving_format(X->get_stride(), inv_var_dim.size(), stride_order)); + inv_var->set_stride(detail::generate_stride(inv_var_dim, stride_order)); + } + } + + // Set scalar tensors + std::vector ones(X->get_dim().size(), 1); + auto infer_scalar_tensors = [&ones](std::shared_ptr& T) { + // Only infer dims and strides if user did not set them + if (T->get_dim().empty()) { + T->set_dim(ones); + } + if (T->get_stride().empty()) { + T->set_stride(ones); + } + }; + infer_scalar_tensors(attributes.inputs[Layernorm_attributes::input_names::EPSILON]); + + return {error_code_t::OK, ""}; + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: " << "Validating LayerNormNode " << attributes.name); + + // Norm forward phase should be set + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.forward_phase == NormFwdPhase_t::NOT_SET, + error_code_t::ATTRIBUTE_NOT_SET, + "Forward phase not set of layernorm node."); + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set& uids_involved_in_operations, + std::vector>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: " << "Building LayerNormNode operations " << attributes.name << " "); + + // Create operation by directly calling cuDNN backend API + Operation_v8 layernorm_operation; + + _CUDNN_CHECK_CUDNN_ERROR( + layernorm_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_NORM_FORWARD_DESCRIPTOR)); + + // Set norm mode + cudnnBackendNormMode_t cudnn_norm_mode; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormMode_t::LAYER_NORM, cudnn_norm_mode)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(layernorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_MODE, + CUDNN_TYPE_NORM_MODE, + 1, + &cudnn_norm_mode)); + + // Set forward phase + cudnnBackendNormFwdPhase_t cudnn_norm_fwd_phase; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.forward_phase, cudnn_norm_fwd_phase)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(layernorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_PHASE, + CUDNN_TYPE_NORM_FWD_PHASE, + 1, + &cudnn_norm_fwd_phase)); + + // Set input tensor X + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Layernorm_attributes::input_names::X); + auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(layernorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &x_desc)); + + // Set scale and bias tensors + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SCALE, Layernorm_attributes::input_names::SCALE); + auto scale_desc = tensors.at(SCALE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(layernorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_SCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &scale_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(BIAS, Layernorm_attributes::input_names::BIAS); + auto bias_desc = tensors.at(BIAS->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(layernorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_BIAS_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &bias_desc)); + + // Set epsilon tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(EPSILON, Layernorm_attributes::input_names::EPSILON); + auto epsilon_desc = tensors.at(EPSILON->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(layernorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_EPSILON_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &epsilon_desc)); + + // Set output tensor Y + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(Y, Layernorm_attributes::output_names::Y); + auto y_desc = tensors.at(Y->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(layernorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_YDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &y_desc)); + + // Set mean and inv_variance for training phase + if (attributes.forward_phase == NormFwdPhase_t::TRAINING) { + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(MEAN, Layernorm_attributes::output_names::MEAN); + auto mean_desc = tensors.at(MEAN->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(layernorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_MEAN_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &mean_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(INV_VARIANCE, Layernorm_attributes::output_names::INV_VARIANCE); + auto inv_var_desc = tensors.at(INV_VARIANCE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(layernorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_INV_VARIANCE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &inv_var_desc)); + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(layernorm_operation.get_raw_desc())); + + operations.push_back(std::make_shared(std::move(layernorm_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "LAYER_NORM"})"_json); + } +#endif +}; + +} // namespace graph + +} // namespace cudnn_frontend \ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/matmul.h b/deps/cudnn-frontend/include/cudnn_frontend/node/matmul.h new file mode 100644 index 0000000..f09d341 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/matmul.h @@ -0,0 +1,253 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend::graph { + +class MatmulNode : public NodeCRTP { + public: + Matmul_attributes attributes; + + MatmulNode(Matmul_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::MATMUL; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for matmul node " << attributes.name); + + attributes.fill_from_context(context); + + // Only inferrencing from (A, B) -> C works today. + auto a_tensor = attributes.inputs[Matmul_attributes::input_names::A]; + auto b_tensor = attributes.inputs[Matmul_attributes::input_names::B]; + auto c_tensor = attributes.outputs[Matmul_attributes::output_names::C]; + + auto const a_tensor_dim = a_tensor->get_dim(); + auto const b_tensor_dim = b_tensor->get_dim(); + auto c_tensor_dim = c_tensor->get_dim(); + + // Only infer dims and strides if user did not set them + if (c_tensor_dim.empty()) { + // CHECK_CUDNN_FRONTEND_ERROR(detail::generate_matmul_output_dim(a_tensor_dim, b_tensor_dim, c_tensor_dim)); + + c_tensor_dim.resize(a_tensor_dim.size()); + int64_t gemm_start_dim = a_tensor_dim.size() - 2; + c_tensor_dim[gemm_start_dim] = a_tensor_dim[gemm_start_dim]; // M + c_tensor_dim[gemm_start_dim + 1] = b_tensor_dim[gemm_start_dim + 1]; // N + + // Broadcast the batches + for (int64_t i = 0; i < gemm_start_dim; ++i) { + c_tensor_dim[i] = std::max(a_tensor_dim[i], b_tensor_dim[i]); + } + + c_tensor->set_dim(c_tensor_dim); + } + if (c_tensor->get_stride().empty()) { + auto const& c_dim = c_tensor->get_dim(); + // Default to Col major + auto const& stride_order = detail::generate_row_major_stride_order(c_dim.size()); + c_tensor->set_stride(detail::generate_stride(c_dim, stride_order)); + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set& uids_involved_in_operations, + std::vector>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: " << "Building MatmulNode operations " << attributes.name << " "); + + // Create matmul descriptor by directly calling cuDNN backend API + MatMulDesc_v8 matmul_descriptor; + + _CUDNN_CHECK_CUDNN_ERROR(matmul_descriptor.initialize_managed_backend_pointer(CUDNN_BACKEND_MATMUL_DESCRIPTOR)); + + // Set compute type + cudnnDataType_t cudnn_data_type; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.compute_data_type, cudnn_data_type)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute( + matmul_descriptor.get_raw_desc(), CUDNN_ATTR_MATMUL_COMP_TYPE, CUDNN_TYPE_DATA_TYPE, 1, &cudnn_data_type)); + + // Set padding value if specified +#if (CUDNN_VERSION >= 8900) + if (attributes.padding_value != 0.0) { + double padding_value = attributes.padding_value; + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(matmul_descriptor.get_raw_desc(), + CUDNN_ATTR_MATMUL_PADDING_VALUE, + CUDNN_TYPE_DOUBLE, + 1, + &padding_value)); + } +#endif + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(matmul_descriptor.get_raw_desc())); + CUDNN_FE_LOG_LABEL_ENDL(matmul_descriptor); + + // Create operation by directly calling cuDNN backend API + Operation_v8 matmul_operation; + + _CUDNN_CHECK_CUDNN_ERROR( + matmul_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_MATMUL_DESCRIPTOR)); + + // Set input tensor A + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(A, Matmul_attributes::input_names::A); + auto a_desc = tensors.at(A->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(matmul_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_MATMUL_ADESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &a_desc)); + + // Set input tensor B + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(B, Matmul_attributes::input_names::B); + auto b_desc = tensors.at(B->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(matmul_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_MATMUL_BDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &b_desc)); + + // Set output tensor C + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(C, Matmul_attributes::output_names::C); + auto c_desc = tensors.at(C->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(matmul_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_MATMUL_CDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &c_desc)); + + // Set matmul descriptor + auto matmul_desc_ptr = matmul_descriptor.get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(matmul_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_MATMUL_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &matmul_desc_ptr)); + + // Set optional override tensors + auto M_override = attributes.inputs.find(Matmul_attributes::input_names::M_override); + if ((M_override != attributes.inputs.end()) && (M_override->second != nullptr)) { + auto m_override_desc = tensors.at(M_override->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(matmul_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_MATMUL_GEMM_M_OVERRIDE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &m_override_desc)); + } + + auto N_override = attributes.inputs.find(Matmul_attributes::input_names::N_override); + if ((N_override != attributes.inputs.end()) && (N_override->second != nullptr)) { + auto n_override_desc = tensors.at(N_override->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(matmul_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_MATMUL_GEMM_N_OVERRIDE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &n_override_desc)); + } + + auto K_override = attributes.inputs.find(Matmul_attributes::input_names::K_override); + if ((K_override != attributes.inputs.end()) && (K_override->second != nullptr)) { + auto k_override_desc = tensors.at(K_override->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(matmul_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_MATMUL_GEMM_K_OVERRIDE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &k_override_desc)); + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(matmul_operation.get_raw_desc())); + + operations.push_back(std::make_shared(std::move(matmul_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "MATMUL"})"_json); + } +#endif +}; + +inline void +INode::matmul(std::shared_ptr a, + std::shared_ptr b, + Matmul_attributes attributes, + std::shared_ptr c) { + attributes.inputs[Matmul_attributes::input_names::A] = a; + attributes.inputs[Matmul_attributes::input_names::B] = b; + attributes.outputs[Matmul_attributes::output_names::C] = c; + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); +} + +inline std::shared_ptr +INode::matmul(std::shared_ptr a, + std::shared_ptr b, + Matmul_attributes attributes) { + if (attributes.name.empty()) { + attributes.name += std::to_string(sub_nodes.size()); + } + attributes.inputs[Matmul_attributes::input_names::A] = a; + attributes.inputs[Matmul_attributes::input_names::B] = b; + + if (a->get_name().empty()) { + a->set_name(attributes.name + "::A"); + }; + if (b->get_name().empty()) { + b->set_name(attributes.name + "::B"); + }; + + auto m_override = attributes.inputs.find(Matmul_attributes::input_names::M_override); + auto n_override = attributes.inputs.find(Matmul_attributes::input_names::N_override); + auto k_override = attributes.inputs.find(Matmul_attributes::input_names::K_override); + + if (m_override != attributes.inputs.end()) { + auto tensor = m_override->second; + if (tensor && tensor->get_name().empty()) { + tensor->set_name(attributes.name + "::M_override"); + } + } + if (n_override != attributes.inputs.end()) { + auto tensor = n_override->second; + if (tensor && tensor->get_name().empty()) { + tensor->set_name(attributes.name + "::N_override"); + } + } + if (k_override != attributes.inputs.end()) { + auto tensor = k_override->second; + if (tensor && tensor->get_name().empty()) { + tensor->set_name(attributes.name + "::K_override"); + } + } + + auto C = attributes.outputs[Matmul_attributes::output_names::C] = output_tensor(attributes.name + "::C"); + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + return C; +} + +} // namespace cudnn_frontend::graph \ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/matmul_fp8.h b/deps/cudnn-frontend/include/cudnn_frontend/node/matmul_fp8.h new file mode 100644 index 0000000..d3fe5e5 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/matmul_fp8.h @@ -0,0 +1,104 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend::graph { + +class MatmulFP8Node : public NodeCRTP { + public: + Matmul_fp8_attributes attributes; + + MatmulFP8Node(Matmul_fp8_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::MATMUL; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for matmul fp8 node " << attributes.name); + + attributes.fill_from_context(context); + + auto const& a_dim = attributes.inputs.at(Matmul_fp8_attributes::input_names::A)->get_dim(); + auto const& b_dim = attributes.inputs.at(Matmul_fp8_attributes::input_names::B)->get_dim(); + auto const& c_dim = attributes.outputs.at(Matmul_fp8_attributes::output_names::C)->get_dim(); + + std::shared_ptr last_output; + + // Matmul + + auto matmul_attributes = Matmul_attributes(); + matmul_attributes.clone_fp8_attributes(attributes); + matmul_attributes.set_name("matmul"); + + last_output = matmul(attributes.inputs.at(Matmul_fp8_attributes::input_names::A), + attributes.inputs.at(Matmul_fp8_attributes::input_names::B), + matmul_attributes); + + // Reduction if GQA for head dimension + if (a_dim.size() == 4 && b_dim.size() == 4 && c_dim.size() == 4 && a_dim[1] == b_dim[1] && + a_dim[1] != c_dim[1] && (a_dim[1] % c_dim[1] == 0)) { + auto gqa_attributes = Reduction_attributes().set_name("gqa_c").set_mode(ReductionMode_t::ADD); + last_output = reduction(last_output, gqa_attributes); + last_output->set_dim(c_dim); + } + + //// Scale Descales + auto mul_attributes = Pointwise_attributes().set_mode(PointwiseMode_t::MUL); + // Descale A + mul_attributes.set_name("descale_a"); + last_output = + pointwise(last_output, attributes.inputs.at(Matmul_fp8_attributes::input_names::Descale_A), mul_attributes); + + // Descale B + mul_attributes.set_name("descale_b"); + last_output = + pointwise(last_output, attributes.inputs.at(Matmul_fp8_attributes::input_names::Descale_B), mul_attributes); + + // Scale C + mul_attributes.set_name("scale_c"); + // Special non-functional-style call. Needed because output already created and provided to user. + pointwise(last_output, + attributes.inputs.at(Matmul_fp8_attributes::input_names::Scale_C), + mul_attributes, + attributes.outputs.at(Matmul_fp8_attributes::output_names::C)); + + // Amax C + auto amax_attributes = Reduction_attributes().set_name("amax_c").set_mode(ReductionMode_t::AMAX); + // Special non-functional-style call. Needed because output already created and provided to user. + reduction(last_output, amax_attributes, attributes.outputs.at(Matmul_fp8_attributes::output_names::Amax_C)); + + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "MATMUL_FP8"})"_json); + } +#endif +}; +inline void +INode::matmul_fp8(std::shared_ptr a, + std::shared_ptr b, + std::shared_ptr descale_a, + std::shared_ptr descale_b, + std::shared_ptr scale_c, + Matmul_fp8_attributes attributes, + std::shared_ptr c, + std::shared_ptr amax_c) { + attributes.inputs[Matmul_fp8_attributes::input_names::A] = a; + attributes.inputs[Matmul_fp8_attributes::input_names::B] = b; + attributes.inputs[Matmul_fp8_attributes::input_names::Descale_A] = descale_a; + attributes.inputs[Matmul_fp8_attributes::input_names::Descale_B] = descale_b; + attributes.inputs[Matmul_fp8_attributes::input_names::Scale_C] = scale_c; + attributes.outputs[Matmul_fp8_attributes::output_names::C] = c; + attributes.outputs[Matmul_fp8_attributes::output_names::Amax_C] = amax_c; + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); +} +} // namespace cudnn_frontend::graph \ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/moe_grouped_matmul.h b/deps/cudnn-frontend/include/cudnn_frontend/node/moe_grouped_matmul.h new file mode 100644 index 0000000..d05d75d --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/moe_grouped_matmul.h @@ -0,0 +1,221 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend::graph { + +class MoeGroupedMatmulNode : public NodeCRTP { + public: + Moe_grouped_matmul_attributes attributes; + + MoeGroupedMatmulNode(Moe_grouped_matmul_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::MOE_GROUPED_MATMUL; + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating MoeGroupedMatmulNode " << attributes.name); + + auto const token_it = attributes.inputs.find(Moe_grouped_matmul_attributes::input_names::Token); + RETURN_CUDNN_FRONTEND_ERROR_IF(token_it == attributes.inputs.end() || token_it->second == nullptr, + error_code_t::ATTRIBUTE_NOT_SET, + "MoeGroupedMatmul input Token not set."); + + auto const weight_it = attributes.inputs.find(Moe_grouped_matmul_attributes::input_names::Weight); + RETURN_CUDNN_FRONTEND_ERROR_IF(weight_it == attributes.inputs.end() || weight_it->second == nullptr, + error_code_t::ATTRIBUTE_NOT_SET, + "MoeGroupedMatmul input Weight not set."); + + auto const first_token_offset_it = + attributes.inputs.find(Moe_grouped_matmul_attributes::input_names::FirstTokenOffset); + RETURN_CUDNN_FRONTEND_ERROR_IF( + first_token_offset_it == attributes.inputs.end() || first_token_offset_it->second == nullptr, + error_code_t::ATTRIBUTE_NOT_SET, + "MoeGroupedMatmul input FirstTokenOffset not set."); + + auto const output_it = attributes.outputs.find(Moe_grouped_matmul_attributes::output_names::Output); + RETURN_CUDNN_FRONTEND_ERROR_IF(output_it == attributes.outputs.end() || output_it->second == nullptr, + error_code_t::ATTRIBUTE_NOT_SET, + "MoeGroupedMatmul output Output not set."); + + return {error_code_t::OK, ""}; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for moe grouped matmul node " << attributes.name); + + attributes.fill_from_context(context); + + auto token_tensor = attributes.inputs[Moe_grouped_matmul_attributes::input_names::Token]; + auto weight_tensor = attributes.inputs[Moe_grouped_matmul_attributes::input_names::Weight]; + auto token_index_tensor = attributes.inputs[Moe_grouped_matmul_attributes::input_names::TokenIndex]; + auto output_tensor = attributes.outputs[Moe_grouped_matmul_attributes::output_names::Output]; + + auto const token_tensor_dim = token_tensor->get_dim(); + auto const weight_tensor_dim = weight_tensor->get_dim(); + auto output_tensor_dim = output_tensor->get_dim(); + + if (output_tensor_dim.empty()) { + output_tensor_dim.resize(3); + output_tensor_dim[0] = 1; + output_tensor_dim[2] = weight_tensor_dim[2]; + if (attributes.mode == MoeGroupedMatmulMode_t::GATHER) { + output_tensor_dim[1] = token_index_tensor->get_dim()[1]; + } else { + output_tensor_dim[1] = token_tensor_dim[1]; + } + output_tensor_dim.resize(3); + + output_tensor->set_dim(output_tensor_dim); + } + + if (output_tensor->get_stride().empty()) { + auto const& output_dim = output_tensor->get_dim(); + auto const& stride_order = detail::generate_row_major_stride_order(output_dim.size()); + output_tensor->set_stride(detail::generate_stride(output_dim, stride_order)); + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set& uids_involved_in_operations, + std::vector>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map>& tensors) const override final { + getLogger() << "[cudnn_frontend] INFO: " + << "Building MoeGroupedMatmulNode operations " << attributes.name << std::endl; + auto cudnn_ver_error = error_t{error_code_t::GRAPH_NOT_SUPPORTED, "Moe grouped matmul requires cuDNN v9.15.0"}; + +#if (CUDNN_VERSION >= 91500) + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(91500, cudnn_ver_error); + CUDNN_FRONTEND_UNUSED(operations); + + auto moe_grouped_matmul_operation = + make_shared_backend_pointer(CUDNN_BACKEND_OPERATION_MOE_GROUPED_MATMUL_DESCRIPTOR); + + cudnnMoeGroupedMatmulMode_t moe_grouped_matmul_mode; + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.mode, moe_grouped_matmul_mode)); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_MODE, + CUDNN_TYPE_MOE_GROUPED_MATMUL_MODE, + 1, + &moe_grouped_matmul_mode)); + + cudnnDataType_t cudnn_data_type; + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.compute_data_type, cudnn_data_type)); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_MATH_PREC, + CUDNN_TYPE_DATA_TYPE, + 1, + &cudnn_data_type)); + + auto token = attributes.inputs.find(Moe_grouped_matmul_attributes::input_names::Token)->second; + auto backend_token = tensors[token->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_TOKEN_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_token)); + + auto weight = attributes.inputs.find(Moe_grouped_matmul_attributes::input_names::Weight)->second; + auto backend_weight = tensors[weight->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_WEIGHT_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_weight)); + + auto first_token_offset = + attributes.inputs.find(Moe_grouped_matmul_attributes::input_names::FirstTokenOffset)->second; + auto backend_first_token_offset = tensors[first_token_offset->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_FIRST_TOKEN_OFFSET_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_first_token_offset)); + + auto output = attributes.outputs.find(Moe_grouped_matmul_attributes::output_names::Output)->second; + auto backend_output = tensors[output->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_OUTPUT_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_output)); + + if (attributes.mode == MoeGroupedMatmulMode_t::GATHER || attributes.mode == MoeGroupedMatmulMode_t::SCATTER) { + auto token_index = attributes.inputs.find(Moe_grouped_matmul_attributes::input_names::TokenIndex)->second; + auto backend_token_index = tensors[token_index->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_TOKEN_INDEX_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_token_index)); + } + + if (attributes.mode == MoeGroupedMatmulMode_t::SCATTER) { + auto token_ks = attributes.inputs.find(Moe_grouped_matmul_attributes::input_names::TokenKs)->second; + auto backend_token_ks = tensors[token_ks->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_TOKEN_KS_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_token_ks)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_TOP_K, + CUDNN_TYPE_INT32, + 1, + &(attributes.top_k))); + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(moe_grouped_matmul_operation->get_backend_descriptor())); + + raw_operations.push_back(moe_grouped_matmul_operation); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; +#else + CUDNN_FRONTEND_UNUSED(uids_involved_in_operations); + CUDNN_FRONTEND_UNUSED(operations); + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FRONTEND_UNUSED(tensors); + return cudnn_ver_error; +#endif + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "MOE_GROUPED_MATMUL"})"_json); + } +#endif +}; + +inline void +INode::moe_grouped_matmul(std::shared_ptr token, + std::shared_ptr weight, + std::shared_ptr first_token_offset, + std::shared_ptr token_index, + std::shared_ptr token_ks, + Moe_grouped_matmul_attributes attributes, + std::shared_ptr output) { + attributes.inputs[Moe_grouped_matmul_attributes::input_names::Token] = token; + attributes.inputs[Moe_grouped_matmul_attributes::input_names::Weight] = weight; + attributes.inputs[Moe_grouped_matmul_attributes::input_names::FirstTokenOffset] = first_token_offset; + attributes.inputs[Moe_grouped_matmul_attributes::input_names::TokenIndex] = token_index; + attributes.inputs[Moe_grouped_matmul_attributes::input_names::TokenKs] = token_ks; + attributes.outputs[Moe_grouped_matmul_attributes::output_names::Output] = output; + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); +} + +} // namespace cudnn_frontend::graph \ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/moe_grouped_matmul_bwd.h b/deps/cudnn-frontend/include/cudnn_frontend/node/moe_grouped_matmul_bwd.h new file mode 100644 index 0000000..2076203 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/moe_grouped_matmul_bwd.h @@ -0,0 +1,181 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend::graph { + +class MoeGroupedMatmulBwdNode : public NodeCRTP { + public: + Moe_grouped_matmul_bwd_attributes attributes; + + MoeGroupedMatmulBwdNode(Moe_grouped_matmul_bwd_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::MOE_GROUPED_MATMUL_BWD; + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating MoeGroupedMatmulBwdNode " << attributes.name); + + auto const doutput_it = attributes.inputs.find(Moe_grouped_matmul_bwd_attributes::input_names::DOutput); + RETURN_CUDNN_FRONTEND_ERROR_IF(doutput_it == attributes.inputs.end() || doutput_it->second == nullptr, + error_code_t::ATTRIBUTE_NOT_SET, + "MoeGroupedMatmulBwd input DOutput not set."); + + auto const token_it = attributes.inputs.find(Moe_grouped_matmul_bwd_attributes::input_names::Token); + RETURN_CUDNN_FRONTEND_ERROR_IF(token_it == attributes.inputs.end() || token_it->second == nullptr, + error_code_t::ATTRIBUTE_NOT_SET, + "MoeGroupedMatmulBwd input Token not set."); + + auto const first_token_offset_it = + attributes.inputs.find(Moe_grouped_matmul_bwd_attributes::input_names::FirstTokenOffset); + RETURN_CUDNN_FRONTEND_ERROR_IF( + first_token_offset_it == attributes.inputs.end() || first_token_offset_it->second == nullptr, + error_code_t::ATTRIBUTE_NOT_SET, + "MoeGroupedMatmulBwd input FirstTokenOffset not set."); + + auto const dweight_it = attributes.outputs.find(Moe_grouped_matmul_bwd_attributes::output_names::DWeight); + RETURN_CUDNN_FRONTEND_ERROR_IF(dweight_it == attributes.outputs.end() || dweight_it->second == nullptr, + error_code_t::ATTRIBUTE_NOT_SET, + "MoeGroupedMatmulBwd output DWeight not set."); + + return {error_code_t::OK, ""}; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for moe grouped matmul bwd node " + << attributes.name); + + attributes.fill_from_context(context); + + auto doutput_tensor = attributes.inputs[Moe_grouped_matmul_bwd_attributes::input_names::DOutput]; + auto token_tensor = attributes.inputs[Moe_grouped_matmul_bwd_attributes::input_names::Token]; + auto token_index_tensor = attributes.inputs[Moe_grouped_matmul_bwd_attributes::input_names::FirstTokenOffset]; + auto dweight_tensor = attributes.outputs[Moe_grouped_matmul_bwd_attributes::output_names::DWeight]; + + auto const doutput_tensor_dim = doutput_tensor->get_dim(); + auto const token_tensor_dim = token_tensor->get_dim(); + auto const token_index_tensor_dim = token_index_tensor->get_dim(); + auto dweight_tensor_dim = dweight_tensor->get_dim(); + + if (dweight_tensor_dim.empty()) { + dweight_tensor_dim.resize(3); + dweight_tensor_dim[0] = token_index_tensor_dim[0]; + dweight_tensor_dim[1] = token_tensor_dim[2]; + dweight_tensor_dim[2] = doutput_tensor_dim[2]; + dweight_tensor->set_dim(dweight_tensor_dim); + } + + if (dweight_tensor->get_stride().empty()) { + auto const& dweight_dim = dweight_tensor->get_dim(); + auto const& stride_order = detail::generate_column_major_stride_order(dweight_dim.size()); + dweight_tensor->set_stride(detail::generate_stride(dweight_dim, stride_order)); + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set& uids_involved_in_operations, + std::vector>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map>& tensors) const override final { + getLogger() << "[cudnn_frontend] INFO: " + << "Building MoeGroupedMatmulBwdNode operations " << attributes.name << std::endl; + auto cudnn_ver_error = + error_t{error_code_t::GRAPH_NOT_SUPPORTED, "Moe grouped matmul bwd requires cuDNN v9.22.0"}; + +#if (CUDNN_VERSION >= 92200) + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(92200, cudnn_ver_error); + CUDNN_FRONTEND_UNUSED(operations); + + auto moe_grouped_matmul_bwd_operation = + make_shared_backend_pointer(CUDNN_BACKEND_OPERATION_MOE_GROUPED_MATMUL_BWD_DESCRIPTOR); + + cudnnDataType_t cudnn_data_type; + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.compute_data_type, cudnn_data_type)); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_bwd_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_BWD_MATH_PREC, + CUDNN_TYPE_DATA_TYPE, + 1, + &cudnn_data_type)); + + auto token = attributes.inputs.find(Moe_grouped_matmul_bwd_attributes::input_names::Token)->second; + auto backend_token = tensors[token->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_bwd_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_BWD_TOKEN_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_token)); + + auto dweight = attributes.outputs.find(Moe_grouped_matmul_bwd_attributes::output_names::DWeight)->second; + auto backend_dweight = tensors[dweight->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_bwd_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_BWD_DWEIGHT_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_dweight)); + + auto first_token_offset = + attributes.inputs.find(Moe_grouped_matmul_bwd_attributes::input_names::FirstTokenOffset)->second; + auto backend_first_token_offset = tensors[first_token_offset->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR( + detail::set_attribute(moe_grouped_matmul_bwd_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_BWD_FIRST_TOKEN_OFFSET_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_first_token_offset)); + + auto doutput = attributes.inputs.find(Moe_grouped_matmul_bwd_attributes::input_names::DOutput)->second; + auto backend_doutput = tensors[doutput->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(moe_grouped_matmul_bwd_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_MOE_GROUPED_MATMUL_BWD_DOUTPUT_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_doutput)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(moe_grouped_matmul_bwd_operation->get_backend_descriptor())); + + raw_operations.push_back(moe_grouped_matmul_bwd_operation); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; +#else + CUDNN_FRONTEND_UNUSED(uids_involved_in_operations); + CUDNN_FRONTEND_UNUSED(operations); + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FRONTEND_UNUSED(tensors); + return cudnn_ver_error; +#endif + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "MOE_GROUPED_MATMUL_BWD"})"_json); + } +#endif +}; + +inline void +INode::moe_grouped_matmul_bwd(std::shared_ptr doutput, + std::shared_ptr token, + std::shared_ptr first_token_offset, + Moe_grouped_matmul_bwd_attributes attributes, + std::shared_ptr dweight) { + attributes.inputs[Moe_grouped_matmul_bwd_attributes::input_names::DOutput] = doutput; + attributes.inputs[Moe_grouped_matmul_bwd_attributes::input_names::Token] = token; + attributes.inputs[Moe_grouped_matmul_bwd_attributes::input_names::FirstTokenOffset] = first_token_offset; + attributes.outputs[Moe_grouped_matmul_bwd_attributes::output_names::DWeight] = dweight; + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); +} + +} // namespace cudnn_frontend::graph \ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/paged_cache_load.h b/deps/cudnn-frontend/include/cudnn_frontend/node/paged_cache_load.h new file mode 100644 index 0000000..9fc8610 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/paged_cache_load.h @@ -0,0 +1,153 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +#include "pointwise.h" +#include "reduction.h" + +namespace cudnn_frontend::graph { + +class PagedCacheLoadNode : public NodeCRTP { + public: + PagedCacheLoad_attributes attributes; + + PagedCacheLoadNode(PagedCacheLoad_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::PAGED_CACHE_LOAD; + } + + error_t + create_cudnn_operations( + std::unordered_set& uids_involved_in_operations, + std::vector>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: " << "Building PagedCacheLoadNode operations " << attributes.name << " "); + auto cudnn_ver_error = error_t{error_code_t::GRAPH_NOT_SUPPORTED, "Paged cache load requires cuDNN v9.5.0"}; + +#if (CUDNN_VERSION >= 90500) + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(90500, cudnn_ver_error); + + // Create operation by directly calling cuDNN backend API + Operation_v8 paged_cache_load_operation; + + _CUDNN_CHECK_CUDNN_ERROR(paged_cache_load_operation.initialize_managed_backend_pointer( + CUDNN_BACKEND_OPERATION_PAGED_CACHE_LOAD_DESCRIPTOR)); + + // Set container tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(container, PagedCacheLoad_attributes::input_names::container); + auto container_desc = tensors.at(container->second->get_uid())->get_raw_desc(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(paged_cache_load_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_PAGED_CACHE_LOAD_CONTAINER_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &container_desc)); + + // Set page table tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(pageTable, PagedCacheLoad_attributes::input_names::pageTable); + auto page_table_desc = tensors.at(pageTable->second->get_uid())->get_raw_desc(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(paged_cache_load_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_PAGED_CACHE_LOAD_PAGE_TABLE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &page_table_desc)); + + // Set sequence length tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(seqLen, PagedCacheLoad_attributes::input_names::seqLen); + auto seq_len_desc = tensors.at(seqLen->second->get_uid())->get_raw_desc(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(paged_cache_load_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_PAGED_CACHE_LOAD_SEQUENCE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &seq_len_desc)); + + // Set output tensor Y + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(yOut, PagedCacheLoad_attributes::output_names::yOut); + auto y_desc = tensors.at(yOut->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(paged_cache_load_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_PAGED_CACHE_LOAD_YDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &y_desc)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(paged_cache_load_operation.get_raw_desc())); + + operations.push_back(std::make_shared(std::move(paged_cache_load_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + + return {error_code_t::OK, ""}; +#else + CUDNN_FRONTEND_UNUSED(uids_involved_in_operations); + CUDNN_FRONTEND_UNUSED(operations); + CUDNN_FRONTEND_UNUSED(tensors); + return cudnn_ver_error; +#endif + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating PagedCacheLoadNode " << attributes.name); + + RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 90500 || detail::get_compiled_version() < 90500, + error_code_t::CUDNN_BACKEND_API_FAILED, + "The cuDNN backend version must be at least 9.5.0 at compile time and runtime " + "in order to use PagedCacheLoadNode."); + + auto const yOut_dims = attributes.outputs.at(PagedCacheLoad_attributes::output_names::yOut)->get_dim(); + auto const yOut_strides = attributes.outputs.at(PagedCacheLoad_attributes::output_names::yOut)->get_stride(); + auto const container_dims = attributes.inputs.at(PagedCacheLoad_attributes::input_names::container)->get_dim(); + auto const blockTable_dims = attributes.inputs.at(PagedCacheLoad_attributes::input_names::pageTable)->get_dim(); + + // In the backend, the k-cache is passed as K^T and has dims [B,H,D,S], while v-cache has dims [B,H,S,D] + // Use the strides to distinguish. + auto yIsTransposed = yOut_strides[2] == 1; + auto s_kv = !yIsTransposed ? yOut_dims[2] : yOut_dims[3]; + + auto block_size = container_dims[2]; + auto block_table_size = blockTable_dims[2]; + bool is_block_table_packed = + attributes.inputs.at(PagedCacheLoad_attributes::input_names::pageTable)->get_ragged_offset() != nullptr; + + RETURN_CUDNN_FRONTEND_ERROR_IF( + !is_block_table_packed && (s_kv + (block_size - 1)) / block_size != block_table_size, + error_code_t::INVALID_VALUE, + "Paged cache load: block table size must equal ceil(s_kv/block_size), except when using packed block " + "tables"); + + return {error_code_t::OK, ""}; + } + + error_t + infer_properties_node() override final { + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + } +#endif +}; + +inline void +INode::paged_cache_load(std::shared_ptr container, + std::shared_ptr seqLen, + std::shared_ptr pageTable, + PagedCacheLoad_attributes attributes, + std::shared_ptr yOut) { + attributes.inputs[PagedCacheLoad_attributes::input_names::container] = std::move(container); + attributes.inputs[PagedCacheLoad_attributes::input_names::seqLen] = std::move(seqLen); + attributes.inputs[PagedCacheLoad_attributes::input_names::pageTable] = std::move(pageTable); + attributes.outputs[PagedCacheLoad_attributes::output_names::yOut] = std::move(yOut); + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); +} +} // namespace cudnn_frontend::graph \ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/pointwise.h b/deps/cudnn-frontend/include/cudnn_frontend/node/pointwise.h new file mode 100644 index 0000000..6d2da14 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/pointwise.h @@ -0,0 +1,390 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend::graph { + +class PointwiseNode : public NodeCRTP { + public: + Pointwise_attributes attributes; + + PointwiseNode(Pointwise_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::POINTWISE; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for pointwise node " << attributes.name); + + attributes.fill_from_context(context); + + auto out_0_tensor = attributes.outputs.at(Pointwise_attributes::output_names::OUT_0); + + auto output_dim = out_0_tensor->get_dim(); + // Only infer dims and strides if user did not set them + if (output_dim.empty()) { + std::vector> input_shapes; + for (const auto& [input_name, input_tensor] : attributes.inputs) { + if (!input_tensor) { + continue; + } + input_shapes.push_back(input_tensor->get_dim()); + } + + CHECK_CUDNN_FRONTEND_ERROR(detail::compute_broadcast_shape(input_shapes, output_dim)); + out_0_tensor->set_dim(output_dim); + } + + if (out_0_tensor->get_stride().empty()) { + for (const auto& [input_name, input_tensor] : attributes.inputs) { + if (input_tensor == nullptr) { + continue; + } + if (input_tensor->get_dim() == out_0_tensor->get_dim()) { + CUDNN_FE_LOG_LABEL_ENDL("INFO:" << " " << out_0_tensor->get_name() + << " stride computed from " << input_tensor->get_name()); + out_0_tensor->set_stride(input_tensor->get_stride()); + break; + } + } + if (out_0_tensor->get_stride().empty() && out_0_tensor->get_is_virtual()) { + // If the tensor is virtual the strides are immaterial + auto input_stride = attributes.inputs.at(Pointwise_attributes::input_names::IN_0)->get_stride(); + std::vector stride_order; + CHECK_CUDNN_FRONTEND_ERROR( + detail::generate_stride_order_preserving_format(input_stride, output_dim.size(), stride_order)); + out_0_tensor->set_stride(detail::generate_stride(output_dim, stride_order)); + } + RETURN_CUDNN_FRONTEND_ERROR_IF(out_0_tensor->get_stride().empty(), + error_code_t::SHAPE_DEDUCTION_FAILED, + "Pointwise output strides could not be computed"); + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set& uids_involved_in_operations, + std::vector>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: " << "Building PointwiseNode operations " << attributes.name << " "); + + // Create pointwise descriptor by directly calling cuDNN backend API + PointWiseDesc_v8 pointwise_descriptor; + + _CUDNN_CHECK_CUDNN_ERROR( + pointwise_descriptor.initialize_managed_backend_pointer(CUDNN_BACKEND_POINTWISE_DESCRIPTOR)); + + // Set pointwise mode + cudnnPointwiseMode_t cudnn_pointwise_mode; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.mode, cudnn_pointwise_mode)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_descriptor.get_raw_desc(), + CUDNN_ATTR_POINTWISE_MODE, + CUDNN_TYPE_POINTWISE_MODE, + 1, + &cudnn_pointwise_mode)); + + // Set compute type + cudnnDataType_t cudnn_data_type; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.compute_data_type, cudnn_data_type)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_descriptor.get_raw_desc(), + CUDNN_ATTR_POINTWISE_MATH_PREC, + CUDNN_TYPE_DATA_TYPE, + 1, + &cudnn_data_type)); + + // Set mode-specific attributes + if (attributes.mode == PointwiseMode_t::RELU_FWD || attributes.mode == PointwiseMode_t::RELU_BWD) { + cudnnNanPropagation_t nan_propagation = CUDNN_PROPAGATE_NAN; + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_descriptor.get_raw_desc(), + CUDNN_ATTR_POINTWISE_NAN_PROPAGATION, + CUDNN_TYPE_NAN_PROPOGATION, + 1, + &nan_propagation)); + + double lower_clip = attributes.relu_lower_clip.value_or(0.0); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_descriptor.get_raw_desc(), + CUDNN_ATTR_POINTWISE_RELU_LOWER_CLIP, + CUDNN_TYPE_DOUBLE, + 1, + &lower_clip)); + + double upper_clip = attributes.relu_upper_clip.value_or(std::numeric_limits::max()); + if (attributes.compute_data_type == DataType_t::FLOAT) { + upper_clip = std::min(upper_clip, std::numeric_limits::max()); + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_descriptor.get_raw_desc(), + CUDNN_ATTR_POINTWISE_RELU_UPPER_CLIP, + CUDNN_TYPE_DOUBLE, + 1, + &upper_clip)); + + double lower_clip_slope = attributes.relu_lower_clip_slope.value_or(0.0); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_descriptor.get_raw_desc(), + CUDNN_ATTR_POINTWISE_RELU_LOWER_CLIP_SLOPE, + CUDNN_TYPE_DOUBLE, + 1, + &lower_clip_slope)); + } else if (attributes.mode == PointwiseMode_t::ELU_FWD || attributes.mode == PointwiseMode_t::ELU_BWD) { + double elu_alpha = attributes.elu_alpha.value_or(1.0); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute( + pointwise_descriptor.get_raw_desc(), CUDNN_ATTR_POINTWISE_ELU_ALPHA, CUDNN_TYPE_DOUBLE, 1, &elu_alpha)); + } else if (attributes.mode == PointwiseMode_t::SOFTPLUS_FWD || + attributes.mode == PointwiseMode_t::SOFTPLUS_BWD) { + double softplus_beta = attributes.softplus_beta.value_or(1.0); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_descriptor.get_raw_desc(), + CUDNN_ATTR_POINTWISE_SOFTPLUS_BETA, + CUDNN_TYPE_DOUBLE, + 1, + &softplus_beta)); + } else if (attributes.mode == PointwiseMode_t::SWISH_FWD || attributes.mode == PointwiseMode_t::SWISH_BWD) { + double swish_beta = attributes.swish_beta.value_or(1.0); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_descriptor.get_raw_desc(), + CUDNN_ATTR_POINTWISE_SWISH_BETA, + CUDNN_TYPE_DOUBLE, + 1, + &swish_beta)); + } else if (attributes.mode == PointwiseMode_t::GEN_INDEX) { + int64_t axis = attributes.get_axis().value_or(-1); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute( + pointwise_descriptor.get_raw_desc(), CUDNN_ATTR_POINTWISE_AXIS, CUDNN_TYPE_INT64, 1, &axis)); + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(pointwise_descriptor.get_raw_desc())); + CUDNN_FE_LOG_LABEL_ENDL(pointwise_descriptor); + + // Create operation by directly calling cuDNN backend API + Operation_v8 pointwise_operation; + + _CUDNN_CHECK_CUDNN_ERROR( + pointwise_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR)); + + // Set the pointwise descriptor + auto pw_desc_ptr = pointwise_descriptor.get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_POINTWISE_PW_DESCRIPTOR, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &pw_desc_ptr)); + + auto const port_count = get_pointwise_mode_port_count(attributes.mode); + bool const is_activation_bwd = detail::is_activation_backward_mode(attributes.mode); + + if (is_activation_bwd) { + // Backward mode: IN_0 is dy, IN_1 is x, OUT_0 is dx + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(IN_0, Pointwise_attributes::input_names::IN_0); + auto dy_desc = tensors.at(IN_0->second->get_uid())->get_raw_desc(); + + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(IN_1, Pointwise_attributes::input_names::IN_1); + auto x_desc = tensors.at(IN_1->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_POINTWISE_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &x_desc)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_POINTWISE_DYDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dy_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(OUT_0, Pointwise_attributes::output_names::OUT_0); + auto dx_desc = tensors.at(OUT_0->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_POINTWISE_DXDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dx_desc)); + } else { + // Forward mode + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(IN_0, Pointwise_attributes::input_names::IN_0); + auto x_desc = tensors.at(IN_0->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_POINTWISE_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &x_desc)); + + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(OUT_0, Pointwise_attributes::output_names::OUT_0); + auto y_desc = tensors.at(OUT_0->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_POINTWISE_YDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &y_desc)); + + if (port_count >= 3) { + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(IN_1, Pointwise_attributes::input_names::IN_1); + auto b_desc = tensors.at(IN_1->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_POINTWISE_BDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &b_desc)); + } + + if (port_count >= 4) { + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(IN_2, Pointwise_attributes::input_names::IN_2); + auto t_desc = tensors.at(IN_2->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(pointwise_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_POINTWISE_TDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &t_desc)); + } + } + + // Set alpha scaling factors (always set to 1.0) + float alpha1 = 1.0f; + float alpha2 = 1.0f; + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute( + pointwise_operation.get_raw_desc(), CUDNN_ATTR_OPERATION_POINTWISE_ALPHA1, CUDNN_TYPE_FLOAT, 1, &alpha1)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute( + pointwise_operation.get_raw_desc(), CUDNN_ATTR_OPERATION_POINTWISE_ALPHA2, CUDNN_TYPE_FLOAT, 1, &alpha2)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(pointwise_operation.get_raw_desc())); + + operations.push_back(std::make_shared(std::move(pointwise_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"({"tag": "POINTWISE"})"_json); + } +#endif +}; + +inline void +INode::pointwise(std::shared_ptr a, + Pointwise_attributes attributes, + std::shared_ptr c) { + attributes.inputs[Pointwise_attributes::input_names::IN_0] = a; + attributes.outputs[Pointwise_attributes::output_names::OUT_0] = c; + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); +} + +inline void +INode::pointwise(std::shared_ptr a, + std::shared_ptr b, + Pointwise_attributes attributes, + std::shared_ptr c) { + attributes.inputs[Pointwise_attributes::input_names::IN_0] = a; + attributes.inputs[Pointwise_attributes::input_names::IN_1] = b; + attributes.outputs[Pointwise_attributes::output_names::OUT_0] = c; + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); +} + +inline void +INode::pointwise(std::shared_ptr a, + std::shared_ptr b, + std::shared_ptr c, + Pointwise_attributes attributes, + std::shared_ptr d) { + attributes.inputs[Pointwise_attributes::input_names::IN_0] = a; + attributes.inputs[Pointwise_attributes::input_names::IN_1] = b; + attributes.inputs[Pointwise_attributes::input_names::IN_2] = c; + attributes.outputs[Pointwise_attributes::output_names::OUT_0] = d; + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); +} + +inline std::shared_ptr +INode::pointwise(std::shared_ptr a, Pointwise_attributes attributes) { + if (attributes.name.empty()) { + attributes.name += std::to_string(sub_nodes.size()); + } + attributes.inputs[Pointwise_attributes::input_names::IN_0] = a; + if (a->get_name().empty()) { + a->set_name(attributes.name + "::IN_0"); + }; + auto OUT_0 = attributes.outputs[Pointwise_attributes::output_names::OUT_0] = + output_tensor(attributes.name + "::OUT_0"); + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + return OUT_0; +} + +inline std::shared_ptr +INode::pointwise(std::shared_ptr a, + std::shared_ptr b, + Pointwise_attributes attributes) { + if (attributes.name.empty()) { + attributes.name += std::to_string(sub_nodes.size()); + } + attributes.inputs[Pointwise_attributes::input_names::IN_0] = a; + attributes.inputs[Pointwise_attributes::input_names::IN_1] = b; + if (a->get_name().empty()) { + a->set_name(attributes.name + "::IN_0"); + }; + if (b->get_name().empty()) { + b->set_name(attributes.name + "::IN_1"); + }; + auto OUT_0 = attributes.outputs[Pointwise_attributes::output_names::OUT_0] = + output_tensor(attributes.name + "::OUT_0"); + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + return OUT_0; +} + +inline std::shared_ptr +INode::pointwise(std::shared_ptr a, + std::shared_ptr b, + std::shared_ptr c, + Pointwise_attributes attributes) { + if (attributes.name.empty()) { + attributes.name += std::to_string(sub_nodes.size()); + } + attributes.inputs[Pointwise_attributes::input_names::IN_0] = a; + attributes.inputs[Pointwise_attributes::input_names::IN_1] = b; + attributes.inputs[Pointwise_attributes::input_names::IN_2] = c; + if (a->get_name().empty()) { + a->set_name(attributes.name + "::IN_0"); + }; + if (b->get_name().empty()) { + b->set_name(attributes.name + "::IN_1"); + }; + if (c->get_name().empty()) { + c->set_name(attributes.name + "::IN_2"); + }; + auto OUT_0 = attributes.outputs[Pointwise_attributes::output_names::OUT_0] = + output_tensor(attributes.name + "::OUT_0"); + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + return OUT_0; +} +} // namespace cudnn_frontend::graph \ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/reduction.h b/deps/cudnn-frontend/include/cudnn_frontend/node/reduction.h new file mode 100644 index 0000000..5944b69 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/reduction.h @@ -0,0 +1,232 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend::graph { + +class ReductionNode : public NodeCRTP { + public: + Reduction_attributes attributes; + + ReductionNode(Reduction_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::REDUCTION; + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating ReductionNode " << attributes.name); + + if (attributes.get_is_deterministic() && detail::get_backend_version() < 91100) { + return {error_code_t::GRAPH_NOT_SUPPORTED, "DETERMINISTIC mode is not supported in cudnn version < 9.11.0"}; + } + return {error_code_t::OK, ""}; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for reduction node " << attributes.name); + + attributes.fill_from_context(context); + + // Only inferrencing from IN_0 to OUT_0 works today. + auto x_tensor = attributes.inputs[Reduction_attributes::input_names::X]; + auto y_tensor = attributes.outputs[Reduction_attributes::output_names::Y]; + + auto const& x_tensor_dim = x_tensor->get_dim(); + auto y_tensor_dim = y_tensor->get_dim(); + // Only infer dims and strides if user did not set them + if (y_tensor_dim.empty()) { + y_tensor->set_dim(x_tensor_dim); + } + if (y_tensor->get_stride().empty()) { + auto const& y_dim = y_tensor->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(y_dim.size()); + y_tensor->set_stride(detail::generate_stride(y_dim, stride_order)); + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set& uids_involved_in_operations, + std::vector>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: " << "Building ReductionNode operations " << attributes.name << " "); + + // Create reduction descriptor by directly calling cuDNN backend API + ReductionDesc_v8 reduction_descriptor; + + // 1. Create the backend descriptor + + _CUDNN_CHECK_CUDNN_ERROR( + reduction_descriptor.initialize_managed_backend_pointer(CUDNN_BACKEND_REDUCTION_DESCRIPTOR)); + + // 2. Set compute type attribute + cudnnDataType_t cudnn_data_type; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.compute_data_type, cudnn_data_type)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(reduction_descriptor.get_raw_desc(), + CUDNN_ATTR_REDUCTION_COMP_TYPE, + CUDNN_TYPE_DATA_TYPE, + 1, + &cudnn_data_type)); + + // 3. Set reduction operator attribute + cudnnReduceTensorOp_t cudnn_reduction_mode; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.get_mode().value(), cudnn_reduction_mode)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(reduction_descriptor.get_raw_desc(), + CUDNN_ATTR_REDUCTION_OPERATOR, + CUDNN_TYPE_REDUCTION_OPERATOR_TYPE, + 1, + &cudnn_reduction_mode)); + + // 4. Set deterministic mode if supported +#if (CUDNN_VERSION >= 91100) + if (detail::get_backend_version() >= 91100) { + bool is_deterministic = attributes.get_is_deterministic(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(reduction_descriptor.get_raw_desc(), + CUDNN_ATTR_REDUCTION_IS_DETERMINISTIC, + CUDNN_TYPE_BOOLEAN, + 1, + &is_deterministic)); + } +#endif + + // 5. Finalize the descriptor + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(reduction_descriptor.get_raw_desc())); + CUDNN_FE_LOG_LABEL_ENDL(reduction_descriptor); + + // Create operation by directly calling cuDNN backend API + Operation_v8 reduction_operation; + + // Validate input tensors are set + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Reduction_attributes::input_names::X); + auto Group_offset = attributes.inputs.find(Reduction_attributes::input_names::Group_offset); + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(Y, Reduction_attributes::output_names::Y); + + // 1. Create the backend operation descriptor + + _CUDNN_CHECK_CUDNN_ERROR( + reduction_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_REDUCTION_DESCRIPTOR)); + + // 2. Set the reduction descriptor attribute + auto reduction_desc_ptr = reduction_descriptor.get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(reduction_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_REDUCTION_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &reduction_desc_ptr)); + + // 3. Set the input tensor (X) descriptor attribute + auto x_backend_desc = tensors.at(X->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(reduction_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_REDUCTION_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &x_backend_desc)); + + // 4. Set the output tensor (Y) descriptor attribute + auto y_backend_desc = tensors.at(Y->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(reduction_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_REDUCTION_YDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &y_backend_desc)); + + // 5. Set the optional group offset tensor descriptor + if (Group_offset != attributes.inputs.end() && Group_offset->second != nullptr) { +#if (CUDNN_VERSION >= 92400) + RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 92400, + error_code_t::GRAPH_NOT_SUPPORTED, + "Reduction group_offset is not supported in cudnn version < 9.24.0"); + auto group_offset_backend_desc = tensors.at(Group_offset->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(reduction_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_REDUCTION_GROUP_OFFSET_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &group_offset_backend_desc)); +#else + return {error_code_t::GRAPH_NOT_SUPPORTED, + "Reduction group_offset is not supported in cudnn version < 9.24.0"}; +#endif + } + + // 6. Finalize the operation descriptor + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(reduction_operation.get_raw_desc())); + + operations.push_back(std::make_shared(std::move(reduction_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"({"tag": "REDUCTION"})"_json); + } +#endif +}; + +inline void +INode::reduction(std::shared_ptr a, + Reduction_attributes attributes, + std::shared_ptr c) { + attributes.inputs[Reduction_attributes::input_names::X] = a; + attributes.outputs[Reduction_attributes::output_names::Y] = c; + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); +} + +inline void +INode::reduction(std::shared_ptr a, + std::shared_ptr group_offset, + Reduction_attributes attributes, + std::shared_ptr c) { + attributes.inputs[Reduction_attributes::input_names::X] = a; + attributes.inputs[Reduction_attributes::input_names::Group_offset] = group_offset; + attributes.outputs[Reduction_attributes::output_names::Y] = c; + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); +} + +inline std::shared_ptr +INode::reduction(std::shared_ptr input, Reduction_attributes attributes) { + attributes.inputs[Reduction_attributes::input_names::X] = input; + auto Y = attributes.outputs[Reduction_attributes::output_names::Y] = output_tensor(attributes.name + "::Y"); + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + return Y; +} + +inline std::shared_ptr +INode::reduction(std::shared_ptr input, + std::shared_ptr group_offset, + Reduction_attributes attributes) { + attributes.inputs[Reduction_attributes::input_names::X] = input; + attributes.inputs[Reduction_attributes::input_names::Group_offset] = group_offset; + auto Y = attributes.outputs[Reduction_attributes::output_names::Y] = output_tensor(attributes.name + "::Y"); + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + return Y; +} +} // namespace cudnn_frontend::graph diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/resample.h b/deps/cudnn-frontend/include/cudnn_frontend/node/resample.h new file mode 100644 index 0000000..34e6031 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/resample.h @@ -0,0 +1,291 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend::graph { + +class ResampleNode : public NodeCRTP { + public: + Resample_attributes attributes; + + ResampleNode(Resample_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::RESAMPLE; + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: " << "Validating ResampleNode " << attributes.name); + + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.generate_index.has_value() == false, + error_code_t::ATTRIBUTE_NOT_SET, + "generate_index attribute not set"); + + if (attributes.generate_index.value() == true && attributes.resample_mode == ResampleMode_t::MAXPOOL) { + CUDNN_FE_VALIDATE_OUTPUT_TENSOR(Resample_attributes::output_names::Index); + } + + // Make sure that the mode can be lowered to BE + cudnnResampleMode_t dummy; + RETURN_CUDNN_FRONTEND_ERROR_IF( + detail::convert_to_cudnn_type(attributes.resample_mode, dummy) != CUDNN_STATUS_SUCCESS, + error_code_t::ATTRIBUTE_NOT_SET, + "Invalid resample mode."); + + return {error_code_t::OK, ""}; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for resample node " << attributes.name); + + auto y_tensor = attributes.outputs[Resample_attributes::output_names::Y]; + auto x_tensor = attributes.inputs[Resample_attributes::input_names::X]; + + attributes.fill_from_context(context); + + // If user does not set shape and layout of the output tensor, + // Get it from node attributes + if (y_tensor->get_dim().empty()) { + auto const x_dim = x_tensor->get_dim(); + auto y_dim = y_tensor->get_dim(); + y_dim = x_dim; + + // 2 cause first two dimensions are batch and channels + for (auto dim = 2u; dim < x_dim.size(); ++dim) { + auto spatial_dim = dim - 2u; + y_dim[dim] = + 1 + (x_dim[dim] + attributes.pre_padding[spatial_dim].numerator + + attributes.post_padding[spatial_dim].numerator - attributes.window[spatial_dim].numerator) / + attributes.stride[spatial_dim].numerator; + } + + y_tensor->set_dim(y_dim); + } + + // If layout is not set, generate the strides from layout + if (y_tensor->get_stride().empty()) { + auto const& y_dim = y_tensor->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(y_dim.size()); + y_tensor->set_stride(detail::generate_stride(y_dim, stride_order)); + } + + if (attributes.outputs[Resample_attributes::output_names::Index]) { + auto index_tensor = attributes.outputs[Resample_attributes::output_names::Index]; + index_tensor->set_dim(y_tensor->get_dim()); + + // If layout is not set, generate the strides from layout + if (index_tensor->get_stride().empty()) { + auto const& index_dim = index_tensor->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(index_dim.size()); + index_tensor->set_stride(detail::generate_stride(index_dim, stride_order)); + } + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set& uids_involved_in_operations, + std::vector>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: " << "Building ResampleNode operations " << attributes.name << " "); + + auto number_of_spatial_dim = static_cast(attributes.window.size()); + + // Create resample descriptor by directly calling cuDNN backend API + ResampleDesc_v8 resample_descriptor; + + _CUDNN_CHECK_CUDNN_ERROR( + resample_descriptor.initialize_managed_backend_pointer(CUDNN_BACKEND_RESAMPLE_DESCRIPTOR)); + + // Set resample mode + cudnnResampleMode_t cudnn_resample_mode; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.resample_mode, cudnn_resample_mode)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_descriptor.get_raw_desc(), + CUDNN_ATTR_RESAMPLE_MODE, + CUDNN_TYPE_RESAMPLE_MODE, + 1, + &cudnn_resample_mode)); + + // Set compute type + cudnnDataType_t cudnn_data_type; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.compute_data_type, cudnn_data_type)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_descriptor.get_raw_desc(), + CUDNN_ATTR_RESAMPLE_COMP_TYPE, + CUDNN_TYPE_DATA_TYPE, + 1, + &cudnn_data_type)); + + // Set nan propagation + cudnnNanPropagation_t nan_opt = CUDNN_PROPAGATE_NAN; + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_descriptor.get_raw_desc(), + CUDNN_ATTR_RESAMPLE_NAN_PROPAGATION, + CUDNN_TYPE_NAN_PROPOGATION, + 1, + &nan_opt)); + + // Set padding mode + cudnnPaddingMode_t cudnn_padding_mode; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.padding_mode, cudnn_padding_mode)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_descriptor.get_raw_desc(), + CUDNN_ATTR_RESAMPLE_PADDING_MODE, + CUDNN_TYPE_PADDING_MODE, + 1, + &cudnn_padding_mode)); + + // Set spatial dimensions + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_descriptor.get_raw_desc(), + CUDNN_ATTR_RESAMPLE_SPATIAL_DIMS, + CUDNN_TYPE_INT64, + 1, + &number_of_spatial_dim)); + + // Set window dimensions + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_descriptor.get_raw_desc(), + CUDNN_ATTR_RESAMPLE_WINDOW_DIMS, + CUDNN_TYPE_FRACTION, + number_of_spatial_dim, + attributes.window.data())); + + // Set pre padding + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_descriptor.get_raw_desc(), + CUDNN_ATTR_RESAMPLE_PRE_PADDINGS, + CUDNN_TYPE_FRACTION, + number_of_spatial_dim, + attributes.pre_padding.data())); + + // Set post padding + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_descriptor.get_raw_desc(), + CUDNN_ATTR_RESAMPLE_POST_PADDINGS, + CUDNN_TYPE_FRACTION, + number_of_spatial_dim, + attributes.post_padding.data())); + + // Set strides + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_descriptor.get_raw_desc(), + CUDNN_ATTR_RESAMPLE_STRIDES, + CUDNN_TYPE_FRACTION, + number_of_spatial_dim, + attributes.stride.data())); + + // Finalize the descriptor + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(resample_descriptor.get_raw_desc())); + CUDNN_FE_LOG_LABEL_ENDL(resample_descriptor); + + // Create operation by directly calling cuDNN backend API + Operation_v8 resample_operation; + + _CUDNN_CHECK_CUDNN_ERROR( + resample_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_RESAMPLE_FWD_DESCRIPTOR)); + + // Set input tensor X + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Resample_attributes::input_names::X); + auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_RESAMPLE_FWD_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &x_desc)); + + // Set output tensor Y + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(Y, Resample_attributes::output_names::Y); + auto y_desc = tensors.at(Y->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_RESAMPLE_FWD_YDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &y_desc)); + + // Set alpha and beta + double alpha = 1.0; + double beta = 0.0; + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute( + resample_operation.get_raw_desc(), CUDNN_ATTR_OPERATION_RESAMPLE_FWD_ALPHA, CUDNN_TYPE_DOUBLE, 1, &alpha)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute( + resample_operation.get_raw_desc(), CUDNN_ATTR_OPERATION_RESAMPLE_FWD_BETA, CUDNN_TYPE_DOUBLE, 1, &beta)); + + // Set resample descriptor + auto resample_raw_desc = resample_descriptor.get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_RESAMPLE_FWD_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &resample_raw_desc)); + + // Set index tensor if available + auto index = attributes.outputs.find(Resample_attributes::output_names::Index); + if ((index != attributes.outputs.end()) && (index->second != nullptr)) { + auto idx_desc = tensors.at(index->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(resample_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_RESAMPLE_FWD_IDXDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &idx_desc)); + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(resample_operation.get_raw_desc())); + + operations.push_back(std::make_shared(std::move(resample_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "RESAMPLE"})"_json); + } +#endif +}; + +inline std::array, 2> +INode::resample(std::shared_ptr input, Resample_attributes attributes) { + if (attributes.name.empty()) { + attributes.name += std::to_string(sub_nodes.size()); + } + attributes.inputs[Resample_attributes::input_names::X] = input; + auto Y = attributes.outputs[Resample_attributes::output_names::Y] = output_tensor(attributes.name + "::Y"); + std::shared_ptr Index = nullptr; + if (attributes.generate_index.has_value() && attributes.generate_index.value() == true && + attributes.resample_mode == ResampleMode_t::MAXPOOL) { + Index = attributes.outputs[Resample_attributes::output_names::Index] = + output_tensor(attributes.name + "::Index"); + } + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + return {Y, Index}; +} + +} // namespace cudnn_frontend::graph \ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/reshape.h b/deps/cudnn-frontend/include/cudnn_frontend/node/reshape.h new file mode 100644 index 0000000..e2b5546 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/reshape.h @@ -0,0 +1,156 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend::graph { + +class ReshapeNode : public NodeCRTP { + public: + Reshape_attributes attributes; + + ReshapeNode(Reshape_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::RESHAPE; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for reshape node " << attributes.name); + + auto y_tensor = attributes.outputs[Reshape_attributes::output_names::Y]; + + attributes.fill_from_context(context); + + // If user does not set shape and layout of the output tensor, + // Get it from node attributes + // If layout is not set, generate the strides from layout + + if (y_tensor->get_dim().empty() && attributes.get_dim().size()) { + y_tensor->set_dim(attributes.dim); + } + + // Graph producers may represent scalar reshape outputs + // as rank-0 tensors (dim={}). Without promotion, stride inference below + // would call generate_NHWC_stride_order(0), which is UB, and the cuDNN + // backend rejects rank-0 descriptors regardless. Promote to canonical + // rank-1 length-1 -- value-preserving because a scalar has volume 1 -- + // while keeping the normalization node-local so downstream broadcast/shape + // inference remains intact. + if (y_tensor->get_dim().empty()) { + y_tensor->set_dim({1}); + if (y_tensor->get_stride().empty()) { + y_tensor->set_stride({1}); + } + } + + if (y_tensor->get_stride().empty()) { + if (attributes.get_stride().size()) { + y_tensor->set_stride(attributes.get_stride()); + } else { + auto const& y_dim = y_tensor->get_dim(); + // Default to NHWC for multi-axis tensors. generate_NHWC_stride_order + // indexes stride_order[1] and assumes num_dims >= 2; use row-major + // for scalars (0-D) and vectors (1-D). + const int64_t rank = static_cast(y_dim.size()); + std::vector const stride_order = + rank < 2 ? detail::generate_row_major_stride_order(rank) : detail::generate_NHWC_stride_order(rank); + y_tensor->set_stride(detail::generate_stride(y_dim, stride_order)); + } + } + + if (y_tensor->get_dim().empty() || y_tensor->get_stride().empty()) { + return {error_code_t::SHAPE_DEDUCTION_FAILED, "Reshape node output shape deduction failed"}; + } + + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Reshape_attributes::input_names::X); + auto const& input_data_type = X->second->get_data_type(); + if (y_tensor->get_data_type() == DataType_t::NOT_SET) { + y_tensor->set_data_type(input_data_type); + } else if (attributes.get_reshape_mode() == ReshapeMode_t::LOGICAL) { + // Lexicographic reshape preserves element type; reject inconsistent metadata. + // VIEW_ONLY paths (e.g. SDPA backward) may set Y dtype after reshape to match a consumer. + RETURN_CUDNN_FRONTEND_ERROR_IF( + y_tensor->get_data_type() != input_data_type, + error_code_t::INVALID_VALUE, + "Output and input tensor data types must match for LOGICAL reshape operation."); + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set& uids_involved_in_operations, + std::vector>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: " << "Building ReshapeNode operations " << attributes.name << " "); + + // Create operation by directly calling cuDNN backend API + Operation_v8 reshape_operation; + + _CUDNN_CHECK_CUDNN_ERROR( + reshape_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_RESHAPE_DESCRIPTOR)); + + // Set input tensor X + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Reshape_attributes::input_names::X); + auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(reshape_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_RESHAPE_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &x_desc)); +#if (CUDNN_VERSION >= 92200) + // Set reshape mode + cudnnBackendReshapeMode_t cudnn_reshape_mode; + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.get_reshape_mode(), cudnn_reshape_mode)); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(reshape_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_RESHAPE_MODE, + CUDNN_TYPE_RESHAPE_MODE, + 1, + &cudnn_reshape_mode)); +#endif + // Set output tensor Y + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(Y, Reshape_attributes::output_names::Y); + auto y_desc = tensors.at(Y->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(reshape_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_RESHAPE_YDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &y_desc)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(reshape_operation.get_raw_desc())); + + operations.push_back(std::make_shared(std::move(reshape_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "RESHAPE"})"_json); + } +#endif +}; + +inline std::shared_ptr +INode::reshape(std::shared_ptr input, Reshape_attributes attributes) { + attributes.inputs[Reshape_attributes::input_names::X] = input; + auto Y = attributes.outputs[Reshape_attributes::output_names::Y] = output_tensor(attributes.name + "::Y"); + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + return Y; +} + +} // namespace cudnn_frontend::graph \ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/rmsnorm.h b/deps/cudnn-frontend/include/cudnn_frontend/node/rmsnorm.h new file mode 100644 index 0000000..bc1f37d --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/rmsnorm.h @@ -0,0 +1,406 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend { + +namespace graph { +class RMSNormNode : public NodeCRTP { + public: + Rmsnorm_attributes attributes; + + RMSNormNode(Rmsnorm_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::RMSNORM; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for rmsnorm node " << attributes.name); + + attributes.fill_from_context(context); + + auto X = attributes.inputs[Rmsnorm_attributes::input_names::X]; + auto Y = attributes.outputs[Rmsnorm_attributes::output_names::Y]; + + // Only infer dims and strides if user did not set them + if (Y->get_dim().empty()) { + Y->set_dim(X->get_dim()); + } + if (Y->get_stride().empty()) { + Y->set_stride(X->get_stride()); + } + + if (attributes.forward_phase == NormFwdPhase_t::TRAINING) { + auto inv_var = attributes.outputs[Rmsnorm_attributes::output_names::INV_VARIANCE]; + // Only infer dims and strides if user did not set them + if (inv_var->get_dim().empty()) { + auto inv_var_dim = X->get_dim(); + auto scale = attributes.inputs[Rmsnorm_attributes::input_names::SCALE]; + if (scale->get_dim().empty()) { + // mean inv_var dim is n,1,1,1 + for (size_t i = 1; i < inv_var_dim.size(); i++) { + inv_var_dim[i] = 1; + } + } else { + for (size_t i = 0; i < inv_var_dim.size(); i++) { + if (scale->get_dim()[i] != 1) { + inv_var_dim[i] = 1; + } + } + } + inv_var->set_dim(inv_var_dim); + } + if (inv_var->get_stride().empty()) { + auto const& inv_var_dim = inv_var->get_dim(); + std::vector stride_order; + CHECK_CUDNN_FRONTEND_ERROR( + detail::generate_stride_order_preserving_format(X->get_stride(), inv_var_dim.size(), stride_order)); + inv_var->set_stride(detail::generate_stride(inv_var_dim, stride_order)); + } + } + return {error_code_t::OK, ""}; + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating RMSNormNode " << attributes.name); + + // Norm forward phase should be set + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.forward_phase == NormFwdPhase_t::NOT_SET, + error_code_t::ATTRIBUTE_NOT_SET, + "Forward phase not set of rmsnorm node."); + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set& uids_involved_in_operations, + std::vector>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: Building RMSNormNode operations " << attributes.name << " "); + + // Create operation by directly calling cuDNN backend API + Operation_v8 rmsnorm_operation; + + _CUDNN_CHECK_CUDNN_ERROR( + rmsnorm_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_NORM_FORWARD_DESCRIPTOR)); + + // Set norm mode to RMS_NORM + cudnnBackendNormMode_t cudnn_norm_mode; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormMode_t::RMS_NORM, cudnn_norm_mode)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rmsnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_MODE, + CUDNN_TYPE_NORM_MODE, + 1, + &cudnn_norm_mode)); + + // Set forward phase + cudnnBackendNormFwdPhase_t cudnn_norm_fwd_phase; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.forward_phase, cudnn_norm_fwd_phase)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rmsnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_PHASE, + CUDNN_TYPE_NORM_FWD_PHASE, + 1, + &cudnn_norm_fwd_phase)); + + // Set input tensor X + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Rmsnorm_attributes::input_names::X); + auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rmsnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &x_desc)); + + // Set scale tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SCALE, Rmsnorm_attributes::input_names::SCALE); + auto scale_desc = tensors.at(SCALE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rmsnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_SCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &scale_desc)); + + // Set epsilon tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(EPSILON, Rmsnorm_attributes::input_names::EPSILON); + auto epsilon_desc = tensors.at(EPSILON->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rmsnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_EPSILON_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &epsilon_desc)); + + // Set output tensor Y + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(Y, Rmsnorm_attributes::output_names::Y); + auto y_desc = tensors.at(Y->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rmsnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_YDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &y_desc)); + + // Set inv_variance for training phase + if (attributes.forward_phase == NormFwdPhase_t::TRAINING) { + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(INV_VARIANCE, Rmsnorm_attributes::output_names::INV_VARIANCE); + auto inv_var_desc = tensors.at(INV_VARIANCE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rmsnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_INV_VARIANCE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &inv_var_desc)); + } + + // Set optional bias tensor + auto BIAS = attributes.inputs.find(Rmsnorm_attributes::input_names::BIAS); + if ((BIAS != attributes.inputs.end()) && (BIAS->second != nullptr)) { + auto bias_desc = tensors.at(BIAS->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rmsnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_FWD_BIAS_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &bias_desc)); + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(rmsnorm_operation.get_raw_desc())); + + operations.push_back(std::make_shared(std::move(rmsnorm_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "RMS_NORM"})"_json); + } +#endif +}; + +class DRMSNormNode : public NodeCRTP { + public: + Rmsnorm_backward_attributes attributes; + + DRMSNormNode(Rmsnorm_backward_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::DRMSNorm; + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating DRMSNormNode node " << attributes.name); + + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.use_dbias.has_value() == false, + error_code_t::ATTRIBUTE_NOT_SET, + "DRMSNormNode node needs has_bias(bool) to be called."); + + return {error_code_t::OK, ""}; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for DRMSNorm node " << attributes.name); + + attributes.fill_from_context(context); + + // TODO: Only inferencing from X works today. + auto X = attributes.inputs[Rmsnorm_backward_attributes::input_names::X]; + auto const x_tensor_dim = X->get_dim(); + + auto DY = attributes.inputs[Rmsnorm_backward_attributes::input_names::DY]; + auto dy_tensor_dim = DY->get_dim(); + + // Only infer dims and strides if user did not set them + if (dy_tensor_dim.empty()) { + dy_tensor_dim.resize(x_tensor_dim.size()); + DY->set_dim(x_tensor_dim); + } + if (DY->get_stride().empty()) { + auto const& DY_dim = DY->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(DY_dim.size()); + DY->set_stride(detail::generate_stride(DY_dim, stride_order)); + } + + auto DX = attributes.outputs[Rmsnorm_backward_attributes::output_names::DX]; + auto dx_tensor_dim = DX->get_dim(); + // Only infer dims and strides if user did not set them + if (dx_tensor_dim.empty()) { + dx_tensor_dim.resize(x_tensor_dim.size()); + DX->set_dim(x_tensor_dim); + } + if (DX->get_stride().empty()) { + auto const& DX_dim = DX->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(DX_dim.size()); + DX->set_stride(detail::generate_stride(DX_dim, stride_order)); + } + + auto scale_bias_dim = X->get_dim(); + scale_bias_dim[0] = 1; + + // Set channel length tensors + auto infer_scale_bias_tensors = [&scale_bias_dim](std::shared_ptr& T) { + auto tensor_dim = T->get_dim(); + // Only infer dims and strides if user did not set them + if (tensor_dim.empty()) { + T->set_dim(scale_bias_dim); + } + if (T->get_stride().empty()) { + auto const& T_dim = T->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(T_dim.size()); + T->set_stride(detail::generate_stride(T_dim, stride_order)); + } + }; + + infer_scale_bias_tensors(attributes.outputs[Rmsnorm_backward_attributes::output_names::DSCALE]); + if (attributes.use_dbias.value()) { + infer_scale_bias_tensors(attributes.outputs[Rmsnorm_backward_attributes::output_names::DBIAS]); + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set& uids_involved_in_operations, + std::vector>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: Building DRMSNormNode operations " << attributes.name << " "); + + // Create operation by directly calling cuDNN backend API + Operation_v8 drmsnorm_operation; + + _CUDNN_CHECK_CUDNN_ERROR( + drmsnorm_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_NORM_BACKWARD_DESCRIPTOR)); + + // Set norm mode to RMS_NORM + cudnnBackendNormMode_t cudnn_norm_mode; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(NormMode_t::RMS_NORM, cudnn_norm_mode)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(drmsnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_MODE, + CUDNN_TYPE_NORM_MODE, + 1, + &cudnn_norm_mode)); + + // Set input tensor X + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(X, Rmsnorm_backward_attributes::input_names::X); + auto x_desc = tensors.at(X->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(drmsnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &x_desc)); + + // Set DY tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(DY, Rmsnorm_backward_attributes::input_names::DY); + auto dy_desc = tensors.at(DY->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(drmsnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_DYDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dy_desc)); + + // Set scale tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(SCALE, Rmsnorm_backward_attributes::input_names::SCALE); + auto scale_desc = tensors.at(SCALE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(drmsnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_SCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &scale_desc)); + + // Set inv_variance tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(INV_VARIANCE, Rmsnorm_backward_attributes::input_names::INV_VARIANCE); + auto inv_var_desc = tensors.at(INV_VARIANCE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(drmsnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_INV_VARIANCE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &inv_var_desc)); + + // Set DSCALE output tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DSCALE, Rmsnorm_backward_attributes::output_names::DSCALE); + auto dscale_desc = tensors.at(DSCALE->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(drmsnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_DSCALE_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dscale_desc)); + + // Set optional DBIAS output tensor + if (attributes.use_dbias.value()) { + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DBIAS, Rmsnorm_backward_attributes::output_names::DBIAS); + auto dbias_desc = tensors.at(DBIAS->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(drmsnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_DBIAS_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dbias_desc)); + } + + // Set DX output tensor + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(DX, Rmsnorm_backward_attributes::output_names::DX); + auto dx_desc = tensors.at(DX->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(drmsnorm_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_NORM_BWD_DXDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &dx_desc)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(drmsnorm_operation.get_raw_desc())); + + operations.push_back(std::make_shared(std::move(drmsnorm_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "RMS_NORM_BPROP"})"_json); + } +#endif +}; + +} // namespace graph + +} // namespace cudnn_frontend \ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/rng.h b/deps/cudnn-frontend/include/cudnn_frontend/node/rng.h new file mode 100644 index 0000000..09b762d --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/rng.h @@ -0,0 +1,187 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend::graph { + +class RngNode : public NodeCRTP { + public: + Rng_attributes attributes; + + RngNode(Rng_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::RNG; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for rng node " << attributes.name); + + auto y_tensor = attributes.outputs[Rng_attributes::output_names::Y]; + + attributes.fill_from_context(context); + + // If user does not set shape and layout of the generated tensor, + // Get it from node attributes + // If layout is not set, generate the strides from layout + + if (y_tensor->get_dim().empty() && attributes.get_dim().size()) { + y_tensor->set_dim(attributes.dim); + } + + if (y_tensor->get_stride().empty()) { + if (attributes.get_stride().size()) { + y_tensor->set_stride(attributes.get_stride()); + } else { + auto const& y_dim = y_tensor->get_dim(); + // Default to NHWC + auto const& stride_order = detail::generate_NHWC_stride_order(y_dim.size()); + y_tensor->set_stride(detail::generate_stride(y_dim, stride_order)); + } + } + + if (y_tensor->get_dim().empty() || y_tensor->get_stride().empty()) { + return {error_code_t::SHAPE_DEDUCTION_FAILED, "RNG node output shape deduction failed"}; + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set& uids_involved_in_operations, + std::vector>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FE_LOG_LABEL("INFO: " << "Building RngNode operations " << attributes.name << " "); + + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.get_distribution() != RngDistribution_t::BERNOULLI, + error_code_t::ATTRIBUTE_NOT_SET, + "no other distribution except bernoulli supported."); + + // Create RNG descriptor by directly calling cuDNN backend API + RngDesc_v8 rng_descriptor; + + _CUDNN_CHECK_CUDNN_ERROR(rng_descriptor.initialize_managed_backend_pointer(CUDNN_BACKEND_RNG_DESCRIPTOR)); + + // Set distribution type + cudnnRngDistribution_t cudnn_rng_distribution; + + _CUDNN_CHECK_CUDNN_ERROR(detail::convert_to_cudnn_type(attributes.get_distribution(), cudnn_rng_distribution)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rng_descriptor.get_raw_desc(), + CUDNN_ATTR_RNG_DISTRIBUTION, + CUDNN_TYPE_RNG_DISTRIBUTION, + 1, + &cudnn_rng_distribution)); + + // Set Bernoulli distribution probability + double bernoulli_prob = attributes.get_bernoulli_probability().value(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rng_descriptor.get_raw_desc(), + CUDNN_ATTR_RNG_BERNOULLI_DIST_PROBABILITY, + CUDNN_TYPE_DOUBLE, + 1, + &bernoulli_prob)); + + // Finalize the descriptor + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(rng_descriptor.get_raw_desc())); + CUDNN_FE_LOG_LABEL_ENDL(rng_descriptor); + + // Create operation by directly calling cuDNN backend API + Operation_v8 rng_operation; + + _CUDNN_CHECK_CUDNN_ERROR( + rng_operation.initialize_managed_backend_pointer(CUDNN_BACKEND_OPERATION_RNG_DESCRIPTOR)); + + // Set output tensor Y + CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(Y, Rng_attributes::output_names::Y); + auto y_desc = tensors.at(Y->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute( + rng_operation.get_raw_desc(), CUDNN_ATTR_OPERATION_RNG_YDESC, CUDNN_TYPE_BACKEND_DESCRIPTOR, 1, &y_desc)); + + // Set RNG descriptor + auto rng_raw_desc = rng_descriptor.get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rng_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_RNG_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &rng_raw_desc)); + + if (attributes.seed.has_value()) { + // Set seed as int64_t value + int64_t seed_value = attributes.get_seed().value(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute( + rng_operation.get_raw_desc(), CUDNN_ATTR_OPERATION_RNG_SEED, CUDNN_TYPE_INT64, 1, &seed_value)); + } else { + // Set seed tensor descriptor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(Seed, Rng_attributes::input_names::Seed); + auto seed_desc = tensors.at(Seed->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rng_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_RNG_SEED, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &seed_desc)); + + // Set offset tensor descriptor + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(Offset, Rng_attributes::input_names::Offset); + auto offset_desc = tensors.at(Offset->second->get_uid())->get_raw_desc(); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rng_operation.get_raw_desc(), + CUDNN_ATTR_OPERATION_RNG_OFFSET_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &offset_desc)); + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(rng_operation.get_raw_desc())); + + operations.push_back(std::make_shared(std::move(rng_operation))); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "RNG"})"_json); + } +#endif +}; + +inline void +INode::rng(std::shared_ptr seed, + std::shared_ptr offset, + Rng_attributes attributes, + std::shared_ptr y) { + attributes.inputs[Rng_attributes::input_names::Seed] = seed; + attributes.inputs[Rng_attributes::input_names::Offset] = offset; + attributes.outputs[Rng_attributes::output_names::Y] = y; + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); +} + +inline std::shared_ptr +INode::rng(std::shared_ptr seed, + std::shared_ptr offset, + Rng_attributes attributes) { + attributes.inputs[Rng_attributes::input_names::Seed] = seed; + attributes.inputs[Rng_attributes::input_names::Offset] = offset; + auto Y = attributes.outputs[Rng_attributes::output_names::Y] = output_tensor(attributes.name + "::Y"); + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + return Y; +} +} // namespace cudnn_frontend::graph \ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/rope.h b/deps/cudnn-frontend/include/cudnn_frontend/node/rope.h new file mode 100644 index 0000000..01b05c4 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/rope.h @@ -0,0 +1,146 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend { + +namespace graph { +class RoPENode : public NodeCRTP { + public: + RoPE_attributes attributes; + + RoPENode(RoPE_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::ROPE; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for RoPE node " << attributes.name); + + attributes.fill_from_context(context); + + auto INPUT = attributes.inputs[RoPE_attributes::input_names::INPUT]; + auto OUTPUT = attributes.outputs[RoPE_attributes::output_names::OUTPUT]; + + if (OUTPUT->get_dim().empty()) { + OUTPUT->set_dim(INPUT->get_dim()); + } + if (OUTPUT->get_stride().empty()) { + OUTPUT->set_stride(INPUT->get_stride()); + } + + return {error_code_t::OK, ""}; + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating RoPENode " << attributes.name); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.inputs.find(RoPE_attributes::input_names::INPUT) == attributes.inputs.end(), + error_code_t::ATTRIBUTE_NOT_SET, + "RoPE node requires INPUT tensor."); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.inputs.find(RoPE_attributes::input_names::FREQS) == attributes.inputs.end(), + error_code_t::ATTRIBUTE_NOT_SET, + "RoPE node requires FREQS tensor."); + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set& uids_involved_in_operations, + std::vector>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(operations); + CUDNN_FE_LOG_LABEL("INFO: Building RoPENode operations " << attributes.name << " "); + +#if (CUDNN_VERSION >= 92400) + // Compile- and run-time checks: RoPE op was added in cuDNN 9.24. + RETURN_CUDNN_FRONTEND_ERROR_IF( + detail::get_backend_version() < 92400, + error_code_t::GRAPH_NOT_SUPPORTED, + "CUDNN_BACKEND_OPERATION_ROPE_FWD_DESCRIPTOR is only available starting cuDNN 9.24."); + + auto rope_operation = + make_shared_backend_pointer((cudnnBackendDescriptorType_t)CUDNN_BACKEND_OPERATION_ROPE_FWD_DESCRIPTOR); + + // Set input X + auto X = attributes.inputs.find(RoPE_attributes::input_names::INPUT)->second; + auto backend_x = tensors[X->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rope_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_ROPE_FWD_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_x)); + + // Set freqs + auto FREQS = attributes.inputs.find(RoPE_attributes::input_names::FREQS)->second; + auto backend_freqs = tensors[FREQS->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rope_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_ROPE_FWD_FREQSDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_freqs)); + + // Set output Y + auto Y = attributes.outputs.find(RoPE_attributes::output_names::OUTPUT)->second; + auto backend_y = tensors[Y->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rope_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_ROPE_FWD_YDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_y)); + + // Set output_scale (host fp32 scalar; default 1.0) + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rope_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_ROPE_FWD_OUTPUT_SCALE, + CUDNN_TYPE_FLOAT, + 1, + &attributes.output_scale)); + + // Set rope_dim (int64; 0 = full rotation = input's head_dim) + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rope_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_ROPE_FWD_ROPE_DIM, + CUDNN_TYPE_INT64, + 1, + &attributes.rope_dim)); + + // Finalize + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(rope_operation->get_backend_descriptor())); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + + raw_operations.push_back(rope_operation); + + return {error_code_t::OK, ""}; +#else + CUDNN_FRONTEND_UNUSED(uids_involved_in_operations); + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FRONTEND_UNUSED(tensors); + return {error_code_t::GRAPH_NOT_SUPPORTED, + "CUDNN_BACKEND_OPERATION_ROPE_FWD_DESCRIPTOR is only available starting cuDNN 9.24."}; +#endif + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"({"tag": "ROPE"})"_json); + } +#endif +}; + +} // namespace graph + +} // namespace cudnn_frontend diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/rope_backward.h b/deps/cudnn-frontend/include/cudnn_frontend/node/rope_backward.h new file mode 100644 index 0000000..fd7bf4f --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/rope_backward.h @@ -0,0 +1,143 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend { + +namespace graph { +class RoPEBackwardNode : public NodeCRTP { + public: + RoPE_backward_attributes attributes; + + RoPEBackwardNode(RoPE_backward_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::ROPE_BWD; + } + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferencing properties for RoPE backward node " << attributes.name); + + attributes.fill_from_context(context); + + auto DY = attributes.inputs[RoPE_backward_attributes::input_names::DY]; + auto DX = attributes.outputs[RoPE_backward_attributes::output_names::DX]; + + if (DX->get_dim().empty()) { + DX->set_dim(DY->get_dim()); + } + if (DX->get_stride().empty()) { + DX->set_stride(DY->get_stride()); + } + + return {error_code_t::OK, ""}; + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating RoPEBackwardNode " << attributes.name); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.inputs.find(RoPE_backward_attributes::input_names::DY) == attributes.inputs.end(), + error_code_t::ATTRIBUTE_NOT_SET, + "RoPE backward node requires DY (input gradient) tensor."); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.inputs.find(RoPE_backward_attributes::input_names::FREQS) == attributes.inputs.end(), + error_code_t::ATTRIBUTE_NOT_SET, + "RoPE backward node requires FREQS tensor."); + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set& uids_involved_in_operations, + std::vector>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(operations); + CUDNN_FE_LOG_LABEL("INFO: Building RoPEBackwardNode operations " << attributes.name << " "); + +#if (CUDNN_VERSION >= 92400) + // Compile- and run-time checks: RoPE bwd op was added in cuDNN 9.24. + RETURN_CUDNN_FRONTEND_ERROR_IF( + detail::get_backend_version() < 92400, + error_code_t::GRAPH_NOT_SUPPORTED, + "CUDNN_BACKEND_OPERATION_ROPE_BWD_DESCRIPTOR is only available starting cuDNN 9.24."); + + auto rope_bwd_operation = + make_shared_backend_pointer((cudnnBackendDescriptorType_t)CUDNN_BACKEND_OPERATION_ROPE_BWD_DESCRIPTOR); + + // Set DY (input gradient) + auto DY = attributes.inputs.find(RoPE_backward_attributes::input_names::DY)->second; + auto backend_dy = tensors[DY->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rope_bwd_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_ROPE_BWD_DYDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_dy)); + + // Set freqs + auto FREQS = attributes.inputs.find(RoPE_backward_attributes::input_names::FREQS)->second; + auto backend_freqs = tensors[FREQS->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rope_bwd_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_ROPE_BWD_FREQSDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_freqs)); + + // Set DX (output gradient) + auto DX = attributes.outputs.find(RoPE_backward_attributes::output_names::DX)->second; + auto backend_dx = tensors[DX->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rope_bwd_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_ROPE_BWD_DXDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_dx)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rope_bwd_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_ROPE_BWD_OUTPUT_SCALE, + CUDNN_TYPE_FLOAT, + 1, + &attributes.output_scale)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(rope_bwd_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_ROPE_BWD_ROPE_DIM, + CUDNN_TYPE_INT64, + 1, + &attributes.rope_dim)); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(rope_bwd_operation->get_backend_descriptor())); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + + raw_operations.push_back(rope_bwd_operation); + + return {error_code_t::OK, ""}; +#else + CUDNN_FRONTEND_UNUSED(uids_involved_in_operations); + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FRONTEND_UNUSED(tensors); + return {error_code_t::GRAPH_NOT_SUPPORTED, + "CUDNN_BACKEND_OPERATION_ROPE_BWD_DESCRIPTOR is only available starting cuDNN 9.24."}; +#endif + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"({"tag": "ROPE_BWD"})"_json); + } +#endif +}; + +} // namespace graph + +} // namespace cudnn_frontend diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/scaled_dot_product_flash_attention.h b/deps/cudnn-frontend/include/cudnn_frontend/node/scaled_dot_product_flash_attention.h new file mode 100644 index 0000000..c3c37c5 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/scaled_dot_product_flash_attention.h @@ -0,0 +1,2730 @@ + +#pragma once + +#include +#include + +#include "../../cudnn_frontend_Heuristics.h" +#include "../../cudnn_frontend_Logging.h" + +#include "../graph_helpers.h" +#include "../node_interface.h" + +#include "diagonal_band_mask.h" +#include "matmul.h" +#include "pointwise.h" +#include "reduction.h" +#include "rng.h" +#include "softmax.h" +#include "paged_cache_load.h" +#include "block_scale_dequantize.h" +#include "sdpa_support_surface.h" + +namespace cudnn_frontend::graph { + +// If a 1-D length/offset index tensor [n] is supplied, promote it to the 4-D +// [n, 1, 1, 1] form that the cuDNN backend requires for seq_len / cu_seq_len and +// ragged-offset tensors. No-op if the tensor is unset or is not 1-D. +inline void +promote_1d_index_tensor_to_4d(std::shared_ptr const& tensor) { + if (tensor == nullptr) { + return; + } + auto const& dim = tensor->get_dim(); + auto const& stride = tensor->get_stride(); + if (dim.size() == 1) { + int64_t const n = dim[0]; + int64_t const leading_stride = (stride.size() == 1) ? stride[0] : 1; + tensor->set_dim({n, 1, 1, 1}).set_stride({leading_stride, 1, 1, 1}); + } +} + +namespace attn::score_modifiers { + +// clang-format off +inline float get_negative_inf_value(); + +inline std::shared_ptr causal_mask( + std::shared_ptr graph, + std::shared_ptr attention_score +); + +inline std::shared_ptr bias( + std::shared_ptr graph, + std::shared_ptr attention_score, + std::shared_ptr bias_tensor +); + +inline std::shared_ptr causal_mask_bottom_right( + std::shared_ptr graph, + std::shared_ptr attention_score, + std::shared_ptr seq_len_q, + std::shared_ptr seq_len_kv +); + +inline std::shared_ptr padding_mask( + std::shared_ptr graph, + std::shared_ptr attention_score, + std::shared_ptr seq_len_kv, + std::shared_ptr seq_len_q +); + +inline std::shared_ptr sliding_window_mask( + std::shared_ptr graph, + std::shared_ptr attention_score, + DiagonalAlignment_t diagonal_alignment, + std::optional left_window, + std::optional right_window, + int64_t s_q, + int64_t s_kv, + std::shared_ptr s_q_ptr, + std::shared_ptr s_kv_ptr, + std::shared_ptr cu_s_q_ptr = nullptr, + std::shared_ptr cu_s_kv_ptr = nullptr +); + +inline std::shared_ptr alibi_mask( + std::shared_ptr graph, + std::shared_ptr attention_score, + std::shared_ptr& alibi_slopes, + int64_t h_q, + int64_t& alibi_slopes_size +); + +inline error_t build_operation_subgraph(std::shared_ptr graph); +// clang-format on + +} // namespace attn::score_modifiers + +template +class SDPANodeBase : public NodeCRTP { + protected: + using input_names = SDPA_attributes::input_names; + using output_names = SDPA_attributes::output_names; + + std::shared_ptr rng_output; + std::shared_ptr alibi_slopes; + int64_t alibi_slopes_size = 0; + + // Promote any 1-D seq_len / cu_seq_len / ragged-offset index tensors to the 4-D + // [n, 1, 1, 1] form the cuDNN backend requires (see promote_1d_index_tensor_to_4d). + void + promote_index_tensors_to_4d() { + for (auto& key : + {input_names::SEQ_LEN_Q, input_names::SEQ_LEN_KV, input_names::CU_SEQ_LEN_Q, input_names::CU_SEQ_LEN_KV}) { + auto const it = attributes.inputs.find(key); + if (it != attributes.inputs.end()) { + promote_1d_index_tensor_to_4d(it->second); + } + } + for (auto& [key, value] : attributes.inputs) { + CUDNN_FRONTEND_UNUSED(key); + if (value != nullptr) { + promote_1d_index_tensor_to_4d(value->get_ragged_offset()); + } + } + for (auto& [key, value] : attributes.outputs) { + CUDNN_FRONTEND_UNUSED(key); + if (value != nullptr) { + promote_1d_index_tensor_to_4d(value->get_ragged_offset()); + } + } + } + + public: + SDPA_attributes attributes; + + SDPANodeBase(SDPA_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + bool + is_paged_v() const { + auto page_table_v_it = attributes.inputs.find(input_names::Page_table_V); + return ((page_table_v_it) != attributes.inputs.end() && page_table_v_it->second != nullptr); + } + + bool + is_paged_k() const { + auto page_table_k_it = attributes.inputs.find(input_names::Page_table_K); + return ((page_table_k_it) != attributes.inputs.end() && page_table_k_it->second != nullptr); + } + + bool + has_seq_len_q() const { + auto seq_len_Q_it = attributes.inputs.find(SDPA_attributes::input_names::SEQ_LEN_Q); + return ((seq_len_Q_it) != attributes.inputs.end() && seq_len_Q_it->second != nullptr); + } + + bool + has_seq_len_kv() const { + auto seq_len_KV_it = attributes.inputs.find(SDPA_attributes::input_names::SEQ_LEN_KV); + return ((seq_len_KV_it) != attributes.inputs.end() && seq_len_KV_it->second != nullptr); + } + + bool + has_cu_seq_len_q() const { + auto cu_seq_len_Q_it = attributes.inputs.find(SDPA_attributes::input_names::CU_SEQ_LEN_Q); + return ((cu_seq_len_Q_it) != attributes.inputs.end() && cu_seq_len_Q_it->second != nullptr); + } + + bool + has_cu_seq_len_kv() const { + auto cu_seq_len_KV_it = attributes.inputs.find(SDPA_attributes::input_names::CU_SEQ_LEN_KV); + return ((cu_seq_len_KV_it) != attributes.inputs.end() && cu_seq_len_KV_it->second != nullptr); + } + + // Helper function to detect MXFP8 (microscaling FP8) mode + // MXFP8 uses block-wise scale factors with E8M0 data type and F8_128x4 reordering + // When detected, we use block_scale_dequantize before matmuls instead of pointwise descale after + bool + is_mxfp8_scaling() const { + auto descale_q_it = attributes.inputs.find(input_names::Descale_Q); + if (descale_q_it == attributes.inputs.end() || descale_q_it->second == nullptr) { + return false; + } + auto const& descale_q = descale_q_it->second; + return (descale_q->get_data_type() == DataType_t::FP8_E8M0 && + descale_q->get_reordering_type() == TensorReordering_t::F8_128x4); + } + + // Helper function to infer KV sequence length + // Note that it cannot be run as part of infer_properties_node as + // this is being used in pre_validate_node + int64_t + infer_s_kv() const { + int64_t s_kv = -1; + + auto get_input_dim = [this](const SDPA_attributes::input_names& input_name) { + auto const input_it = attributes.inputs.find(input_name); + if (input_it != attributes.inputs.end()) { + return input_it->second->get_dim(); + } else { + return std::vector({-1, -1, -1, -1}); + } + }; + + auto const& k_dim = get_input_dim(input_names::K); + auto const& v_dim = get_input_dim(input_names::V); + + // If s_kv was set explicitly, use that + if (attributes.max_seq_len_kv.has_value()) { + s_kv = attributes.max_seq_len_kv.value(); + } + // When one of K or V cache are paged, s_kv can be extracted directly + else if (!is_paged_k()) { + s_kv = k_dim[2]; + + } else if (!is_paged_v()) { + s_kv = v_dim[2]; + } else { + CUDNN_FE_LOG_LABEL_ENDL( + "WARNING: maximum kv sequence length is being inferred. To set it explicitly, please use " + "\"set_paged_attention_max_seq_len_kv\""); + + auto bias_it = attributes.inputs.find(input_names::Bias); + auto rng_it = attributes.outputs.find(output_names::RNG_DUMP); + + // If there is a bias, extract it from there + if (bias_it != attributes.inputs.end() && bias_it->second != nullptr) { + s_kv = get_input_dim(input_names::Bias)[3]; + // If there is an rng_dump output, extract it from there + } else if (rng_it != attributes.outputs.end() && rng_it->second != nullptr) { + s_kv = rng_it->second->get_dim()[3]; + // When both caches are paged, and the above failed, we need to infer s_kv from the page table and + // container + } else { + // [b, 1, ceil(s_kv/block_size), 1] + auto page_table_dim_k = get_input_dim(input_names::Page_table_K); + // [b, h_k, block_size, d_k] + auto const container_dim_k = get_input_dim(input_names::K); + int64_t s_k = page_table_dim_k[2] * container_dim_k[2]; + + // [b, 1, ceil(s_kv/block_size), 1] + auto page_table_dim_v = get_input_dim(input_names::Page_table_V); + // [b, h_v, block_size, d_v] + auto const container_dim_v = get_input_dim(input_names::V); + int64_t s_v = page_table_dim_v[2] * container_dim_v[2]; + + s_kv = std::min(s_k, s_v); + } + } + + return s_kv; + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating SDPANode " << attributes.name); + + // check that Q, K, V, O tensors has been assigned + // check that dim and strides has been assigned and last stride is 1 +#define CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(port, port_map) \ + { \ + std::shared_ptr tensor_ptr = port_map.at(port); \ + RETURN_CUDNN_FRONTEND_ERROR_IF(tensor_ptr->get_dim().size() != 4, \ + error_code_t::ATTRIBUTE_NOT_SET, \ + "The dim for " + std::string(#port) + " is invalid"); \ + RETURN_CUDNN_FRONTEND_ERROR_IF(tensor_ptr->get_stride().size() != 4, \ + error_code_t::ATTRIBUTE_NOT_SET, \ + "The stride for " + std::string(#port) + " is invalid"); \ + RETURN_CUDNN_FRONTEND_ERROR_IF( \ + tensor_ptr->get_stride()[3] != 1, \ + error_code_t::GRAPH_NOT_SUPPORTED, \ + "The stride for the last dimension corresponding to the embedding size per head should be 1 for " + \ + std::string(#port)); \ + } + + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::Q, attributes.inputs); + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::K, attributes.inputs); + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::V, attributes.inputs); + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(output_names::O, attributes.outputs); + + if (attributes.generate_stats.value_or(false) == true) { + CUDNN_FE_VALIDATE_OUTPUT_TENSOR(output_names::Stats); + } + + // If max is requested, validate that the output tensor is present + if (attributes.outputs.find(output_names::Max) != attributes.outputs.end() && + attributes.outputs.at(output_names::Max) != nullptr) { + CUDNN_FE_VALIDATE_OUTPUT_TENSOR(output_names::Max); + } + + // If sum_exp is requested, validate that the output tensor is present + if (attributes.outputs.find(output_names::Sum_exp) != attributes.outputs.end() && + attributes.outputs.at(output_names::Sum_exp) != nullptr) { + CUDNN_FE_VALIDATE_OUTPUT_TENSOR(output_names::Sum_exp); + } + +#undef CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE + + // validate backend limitations for the operation + auto validation_result = + attributes.validate_sdpa_support_surface(this->context, infer_s_kv(), is_paged_k(), is_paged_v()); + if (validation_result.is_good() == false) { + return validation_result; + } + + // return NOT_SET if sink_token present with 9.12 and below + RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 91300 && + attributes.inputs.find(input_names::SINK_TOKEN) != attributes.inputs.end(), + error_code_t::ATTRIBUTE_NOT_SET, + "SDPA with sink_token is not supported before 9.13."); + + // Validate MXFP8 scale factors if present + if (is_mxfp8_scaling()) { + // MXFP8 requires cuDNN 9.21.0 or later + RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 92100, + error_code_t::GRAPH_NOT_SUPPORTED, + "MXFP8 SDPA requires cuDNN 9.21.0 or later"); + + // Get device SM version from context + CHECK_CUDNN_FRONTEND_ERROR(this->context.populate_sm_version_from_device()); + int32_t const sm_version = this->context.get_sm_version(); + int32_t const prop_major = sm_version / 10; + + RETURN_CUDNN_FRONTEND_ERROR_IF(10 != prop_major, + error_code_t::GRAPH_NOT_SUPPORTED, + "MXFP8 SDPA is only supported on Blackwell Data Center architectures."); + + auto const& q_dim = attributes.inputs.at(input_names::Q)->get_dim(); + auto const& k_dim = attributes.inputs.at(input_names::K)->get_dim(); + auto const& v_dim = attributes.inputs.at(input_names::V)->get_dim(); + int64_t const b = q_dim[0]; + int64_t const h_q = q_dim[1]; + int64_t const s_q = q_dim[2]; + int64_t const d = q_dim[3]; + int64_t const h_k = k_dim[1]; + int64_t const h_v = v_dim[1]; + int64_t const s_kv = infer_s_kv(); + + // MXFP8 block size is fixed at 32 + constexpr int64_t block_size = 32; + int64_t const d_scale = (d + block_size - 1) / block_size; + int64_t const s_scale = (s_kv + block_size - 1) / block_size; + + // Validate Descale_Q + auto const& descale_q = attributes.inputs.at(input_names::Descale_Q); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_q->get_data_type() != DataType_t::FP8_E8M0, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_Q to have FP8_E8M0 data type"); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_q->get_reordering_type() != TensorReordering_t::F8_128x4, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_Q to have F8_128x4 reordering"); + RETURN_CUDNN_FRONTEND_ERROR_IF( + descale_q->get_stride()[3] != 1, + error_code_t::GRAPH_NOT_SUPPORTED, + "MXFP8 SDPA requires Descale_Q to have contiguous d_scale dimension (stride[3] == 1)"); + + // Validate Descale_K + auto const& descale_k = attributes.inputs.at(input_names::Descale_K); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_k->get_data_type() != DataType_t::FP8_E8M0, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_K to have FP8_E8M0 data type"); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_k->get_reordering_type() != TensorReordering_t::F8_128x4, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_K to have F8_128x4 reordering"); + RETURN_CUDNN_FRONTEND_ERROR_IF( + descale_k->get_stride()[3] != 1, + error_code_t::GRAPH_NOT_SUPPORTED, + "MXFP8 SDPA requires Descale_K to have contiguous d_scale dimension (stride[3] == 1)"); + + // Validate Descale_V + auto const& descale_v = attributes.inputs.at(input_names::Descale_V); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_v->get_data_type() != DataType_t::FP8_E8M0, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_V to have FP8_E8M0 data type"); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_v->get_reordering_type() != TensorReordering_t::F8_128x4, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_V to have F8_128x4 reordering"); + // // SF_V scales along s dimension (not d), so s_scale must be contiguous + // RETURN_CUDNN_FRONTEND_ERROR_IF( + // descale_v->get_stride()[2] != 1, + // error_code_t::GRAPH_NOT_SUPPORTED, + // "MXFP8 SDPA requires Descale_V to have contiguous s_scale dimension (stride[2] == 1)"); + + // Validate dimension consistency for SF_Q: [b, h_q, s_q_padded, d_scale_padded] + auto const& sf_q_dim = descale_q->get_dim(); + RETURN_CUDNN_FRONTEND_ERROR_IF(sf_q_dim[0] != b || sf_q_dim[1] != h_q, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_Q batch/head dimensions must match Q"); + RETURN_CUDNN_FRONTEND_ERROR_IF( + sf_q_dim[3] < d_scale, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_Q d_scale dimension too small (expected >= " + std::to_string(d_scale) + ")"); + + // Validate dimension consistency for SF_K: [b, h_k, s_kv_padded, d_scale_padded] + auto const& sf_k_dim = descale_k->get_dim(); + RETURN_CUDNN_FRONTEND_ERROR_IF(sf_k_dim[0] != b || sf_k_dim[1] != h_k, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_K batch/head dimensions must match K"); + RETURN_CUDNN_FRONTEND_ERROR_IF( + sf_k_dim[3] < d_scale, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_K d_scale dimension too small (expected >= " + std::to_string(d_scale) + ")"); + + // Validate dimension consistency for SF_V: [b, h_v, s_scale_padded, d_padded] + auto const& sf_v_dim = descale_v->get_dim(); + RETURN_CUDNN_FRONTEND_ERROR_IF(sf_v_dim[0] != b || sf_v_dim[1] != h_v, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_V batch/head dimensions must match V"); + RETURN_CUDNN_FRONTEND_ERROR_IF( + sf_v_dim[2] < s_scale, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_V s_scale dimension too small (expected >= " + std::to_string(s_scale) + ")"); + + // Log MXFP8 configuration for debugging + getLogger() << "[cudnn_frontend] INFO: MXFP8 SDPA configuration validated:" << std::endl + << " Q dims: [" << b << ", " << h_q << ", " << s_q << ", " << d << "]" << std::endl + << " SF_Q dims: [" << sf_q_dim[0] << ", " << sf_q_dim[1] << ", " << sf_q_dim[2] << ", " + << sf_q_dim[3] << "]" << std::endl + << " SF_K dims: [" << sf_k_dim[0] << ", " << sf_k_dim[1] << ", " << sf_k_dim[2] << ", " + << sf_k_dim[3] << "]" << std::endl + << " SF_V dims: [" << sf_v_dim[0] << ", " << sf_v_dim[1] << ", " << sf_v_dim[2] << ", " + << sf_v_dim[3] << "]" << std::endl + << " Expected d_scale: " << d_scale << ", s_scale: " << s_scale << std::endl; + } + + return {error_code_t::OK, ""}; + } + + error_t + infer_properties_node() override final { + // If a 1-D length/offset index tensor [n] is supplied, promote it to the 4-D + // [n, 1, 1, 1] (contiguous) form the cuDNN backend requires. + promote_index_tensors_to_4d(); + + if (attributes.generate_stats.value_or(false)) { + auto stats = attributes.outputs.at(output_names::Stats); + auto stats_dim = stats->get_dim(); + + if (stats_dim.empty()) { + // Fill properties of virtual tensors + auto const& p_dim = attributes.inputs[input_names::Q]->get_dim(); + auto b = p_dim[0]; + auto h = p_dim[1]; + auto s_q = p_dim[2]; + stats->set_dim({b, h, s_q, 1}).set_stride({h * s_q, s_q, 1, 1}); + } + } + + if (attributes.outputs[output_names::Max] != nullptr) { + auto max = attributes.outputs.at(output_names::Max); + + if (max->get_dim().empty()) { + // Fill properties of virtual tensors + auto const& p_dim = attributes.inputs[input_names::Q]->get_dim(); + auto b = p_dim[0]; + auto h = p_dim[1]; + auto s_q = p_dim[2]; + max->set_dim({b, h, s_q, 1}).set_stride({h * s_q, s_q, 1, 1}); + } + } + + if (attributes.outputs[output_names::Sum_exp] != nullptr) { + auto sum_exp = attributes.outputs.at(output_names::Sum_exp); + + if (sum_exp->get_dim().empty()) { + // Fill properties of virtual tensors + auto const& p_dim = attributes.inputs[input_names::Q]->get_dim(); + auto b = p_dim[0]; + auto h = p_dim[1]; + auto s_q = p_dim[2]; + sum_exp->set_dim({b, h, s_q, 1}).set_stride({h * s_q, s_q, 1, 1}); + } + } + return {error_code_t::OK, ""}; + } + + error_t + post_validate_node() const override final { +#define CUDNN_FE_VALIDATE_STRIDE(port, port_map) \ + { \ + auto const& t = port_map.find(port); \ + RETURN_CUDNN_FRONTEND_ERROR_IF( \ + t->second->get_stride().back() != 1, \ + error_code_t::GRAPH_NOT_SUPPORTED, \ + "The stride for the last dimension corresponding to the embedding size per head should be 1 for " + \ + std::string(#port)); \ + } + + CUDNN_FE_VALIDATE_STRIDE(output_names::O, attributes.outputs); + +#undef CUDNN_FE_VALIDATE_STRIDE + + return {error_code_t::OK, ""}; + } + + virtual int64_t + get_fe_workspace_size_node() const override final { + int64_t size = 0; + + // align alibi slopes memory to 16 bytes + size += ((alibi_slopes_size + 15) / 16 * 16); + + return size; + } + + virtual error_t + collect_tensors_in_workspace_node( + std::unordered_map>>& + workspace_modifications, + int64_t& offset) const override final { + if (attributes.alibi_mask) { + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(Q, input_names::Q); + int64_t const h_q = Q->second->get_dim()[1]; + auto alibi_slopes_vec = detail::get_alibi_slope(h_q); + workspace_modifications.emplace(alibi_slopes->get_uid(), std::make_tuple(0, offset, alibi_slopes_vec)); + int64_t alibi_slopes_size_padded = ((alibi_slopes_size + 15) / 16 * 16); + offset = offset + alibi_slopes_size_padded; + } + return {error_code_t::OK, ""}; + } + + error_t + collect_tensors_to_dump_node( + std::vector, char>>& tensors_to_dump) const override final { + std::unordered_set seen_uids; + auto add_tensor = [&tensors_to_dump, &seen_uids](std::shared_ptr const& tensor) { + if (tensor == nullptr) { + return; + } + if (seen_uids.insert(tensor->get_uid()).second) { + tensors_to_dump.emplace_back(tensor, 'd'); + } + }; + + auto const seq_len_q_it = attributes.inputs.find(input_names::SEQ_LEN_Q); + if (seq_len_q_it != attributes.inputs.end()) { + add_tensor(seq_len_q_it->second); + } + + auto const seq_len_kv_it = attributes.inputs.find(input_names::SEQ_LEN_KV); + if (seq_len_kv_it != attributes.inputs.end()) { + add_tensor(seq_len_kv_it->second); + } + + auto const cu_seq_len_q_it = attributes.inputs.find(input_names::CU_SEQ_LEN_Q); + if (cu_seq_len_q_it != attributes.inputs.end()) { + add_tensor(cu_seq_len_q_it->second); + } + + auto const cu_seq_len_kv_it = attributes.inputs.find(input_names::CU_SEQ_LEN_KV); + if (cu_seq_len_kv_it != attributes.inputs.end()) { + add_tensor(cu_seq_len_kv_it->second); + } + + for (auto const& tensor : {attributes.inputs.at(input_names::Q), + attributes.inputs.at(input_names::K), + attributes.inputs.at(input_names::V), + attributes.outputs.at(output_names::O)}) { + if (tensor != nullptr) { + add_tensor(tensor->get_ragged_offset()); + } + } + + auto const stats_it = attributes.outputs.find(output_names::Stats); + if (stats_it != attributes.outputs.end() && stats_it->second != nullptr) { + add_tensor(stats_it->second->get_ragged_offset()); + } + + auto const max_it = attributes.outputs.find(output_names::Max); + if (max_it != attributes.outputs.end() && max_it->second != nullptr) { + add_tensor(max_it->second->get_ragged_offset()); + } + + auto const sum_exp_it = attributes.outputs.find(output_names::Sum_exp); + if (sum_exp_it != attributes.outputs.end() && sum_exp_it->second != nullptr) { + add_tensor(sum_exp_it->second->get_ragged_offset()); + } + + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j["is_mxfp8"] = is_mxfp8_scaling(); + j["unfuse_fma"] = attributes.unfuse_fma; + if (auto const rescale_threshold = get_rescale_threshold_from_env(); rescale_threshold.has_value()) { + j["rescale_threshold"] = rescale_threshold.value(); + } + if (is_mxfp8_scaling()) { + j.update(R"({"tag": "SDPA_MXFP8_FWD"})"_json); + } else if (attributes.mma_core_mode == DataType_t::FP8_E4M3 || + attributes.mma_core_mode == DataType_t::FP8_E5M2) { + j.update(R"({"tag": "SDPA_FP8_FWD"})"_json); + } else { + j.update(R"({"tag": "SDPA"})"_json); + } + } +#endif +}; + +class CompositeSDPANode : public SDPANodeBase { + public: + CompositeSDPANode(SDPA_attributes&& attributes_, detail::Context const& context) + : SDPANodeBase(std::move(attributes_), context) {} + + Type + getType() override final { + return Type::COMPOSITE; + } + + error_t + expand_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for CompositeSDPANode node " << attributes.name); + + // DO NOT REMOVE + // input data type is needed for: + // - aType of bmm2 + // - dropout scale in pre 8.9.3 + attributes.fill_from_context(this->context); + + // Gather dim to fill properties of virtual tensors + auto const& q_dim = attributes.inputs[input_names::Q]->get_dim(); + auto b = q_dim[0]; + auto h_q = q_dim[1]; + auto s_q = q_dim[2]; + auto d_qk = q_dim[3]; + auto const& k_dim = attributes.inputs[input_names::K]->get_dim(); + auto h_k = k_dim[1]; + auto const& v_dim = attributes.inputs[input_names::V]->get_dim(); + auto h_v = v_dim[1]; + auto d_v = v_dim[3]; + // Infer s_kv + int64_t s_kv = infer_s_kv(); + + // Check if using MXFP8 (microscaling FP8) with block-wise scale factors + // Need to check this early because MXFP8 dequantization must happen before K transpose + bool const use_mxfp8 = is_mxfp8_scaling(); + + std::shared_ptr k_cache; + if (!is_paged_k()) { + if (use_mxfp8) { + // MXFP8: Transpose K and SF_K first, then dequantize + // The backend expects both K and its scale factor in transposed layout + + // Step 1: Transpose K -> KT by swapping dims and strides + // K = {b, h_k, s_kv, d_qk} -> KT = {b, h_k, d_qk, s_kv} + std::vector kt_dim = attributes.inputs[input_names::K]->get_dim(); + std::vector kt_stride = attributes.inputs[input_names::K]->get_stride(); + std::swap(kt_dim[2], kt_dim[3]); + std::swap(kt_stride[2], kt_stride[3]); + attributes.inputs[input_names::K]->set_dim(kt_dim); + attributes.inputs[input_names::K]->set_stride(kt_stride); + + // Step 2: Transpose SF_K similarly + // SF_K = {b, h_k, s_kv_scale, d_qk_scale} -> {b, h_k, d_qk_scale, s_kv_scale} + auto& sf_k = attributes.inputs[input_names::Descale_K]; + std::vector sf_k_dim = sf_k->get_dim(); + std::vector sf_k_stride = sf_k->get_stride(); + std::swap(sf_k_dim[2], sf_k_dim[3]); + std::swap(sf_k_stride[2], sf_k_stride[3]); + sf_k->set_dim(sf_k_dim); + sf_k->set_stride(sf_k_stride); + + // Step 3: Dequantize transposed K with transposed SF_K + auto dequant_k_attrs = Block_scale_dequantize_attributes() + .set_name("dequant_k") + .set_block_size({1, 32}); // Standard MXFP8 block size + + auto k_dequant = std::make_shared(); + k_dequant->set_is_virtual(true); + k_dequant->set_dim(kt_dim); + k_dequant->set_stride(kt_stride); + + block_scale_dequantize(attributes.inputs[input_names::K], sf_k, dequant_k_attrs, k_dequant); + + k_cache = k_dequant; + } else { + // Non-MXFP8: map K->KT directly + // cuDNN frontend API attention requires Q, K, V where + // Q = {b, h_q, s_q, d_qk} + // K = {b, h_k, s_kv, d_qk} + // V = {b, h_v, s_kv, d_v} + // but cuDNN backend API attention requires Q, KT, V + // Q = {b, h_q, s_q, d_qk} + // KT = {b, h_k, d_qk, s_kv} + // V = {b, h_v, s_kv, d_v} + // So the code below maps the K->KT + std::vector temp_vec; + + temp_vec = attributes.inputs[input_names::K]->get_dim(); + std::swap(temp_vec[2], temp_vec[3]); + attributes.inputs[input_names::K]->set_dim(temp_vec); + + temp_vec = attributes.inputs[input_names::K]->get_stride(); + std::swap(temp_vec[2], temp_vec[3]); + attributes.inputs[input_names::K]->set_stride(temp_vec); + + k_cache = attributes.inputs[input_names::K]; + } + } else { + // Create a paged cache load operation + auto paged_cache_load_attributes_k = PagedCacheLoad_attributes().set_name("paged_k_cache_operation"); + // Need to create virtual tensor descriptor for yOut here as it cannot be inferred + // K-cache has BHDS layout + k_cache = std::make_shared(); + k_cache->set_is_virtual(true); + k_cache->set_dim({b, h_k, d_qk, s_kv}); + k_cache->set_stride({d_qk * s_kv * h_k, d_qk * s_kv, 1, d_qk}); + k_cache->set_data_type(attributes.inputs[input_names::K]->get_data_type()); + paged_cache_load(attributes.inputs[input_names::K], + attributes.inputs[input_names::SEQ_LEN_KV], + attributes.inputs[input_names::Page_table_K], + paged_cache_load_attributes_k, + k_cache); + } + + // This tensor tracks the main chain of data flow + std::shared_ptr last_output; + + // Prepare Q and K for first matmul + // For MXFP8: Q is dequantized here, K was dequantized above before transpose + // For regular FP8: use raw tensors, descale applied after matmul + std::shared_ptr q_for_bmm1; + std::shared_ptr k_for_bmm1; + + if (use_mxfp8) { + // MXFP8: Dequantize Q using block scale factors + auto dequant_q_attrs = Block_scale_dequantize_attributes() + .set_name("dequant_q") + .set_block_size({1, 32}); // Standard MXFP8 block size + auto q_dequant = std::make_shared(); + q_dequant->set_is_virtual(true); + q_dequant->set_dim(attributes.inputs[input_names::Q]->get_dim()); + q_dequant->set_stride(attributes.inputs[input_names::Q]->get_stride()); + block_scale_dequantize(attributes.inputs[input_names::Q], + attributes.inputs[input_names::Descale_Q], + dequant_q_attrs, + q_dequant); + q_for_bmm1 = q_dequant; + + // K was already dequantized and transposed above + k_for_bmm1 = k_cache; + } else { + q_for_bmm1 = attributes.inputs[input_names::Q]; + k_for_bmm1 = k_cache; + } + + //// Q * K + auto bmm1_attributes = Matmul_attributes() + .set_name("bmm1") + .set_m_override(attributes.inputs[input_names::SEQ_LEN_Q]) + .set_n_override(attributes.inputs[input_names::SEQ_LEN_KV]); + + if (attributes.padding_mask) { + bmm1_attributes.set_padding(0.0); + } + + auto const& bmm1_output = matmul(q_for_bmm1, k_for_bmm1, bmm1_attributes); + // Setting dim and strides as pointwise op wont have knowledge of how to do it for mha. + bmm1_output->set_dim({b, h_q, s_q, s_kv}).set_stride({h_q * s_q * s_kv, s_q * s_kv, s_kv, 1}); + last_output = bmm1_output; + + //// Optional Attn scale + // In case user provided a scalar value, do a fused scalar. + if (attributes.attn_scale_value.has_value()) { + attributes.inputs[input_names::Attn_scale] = + std::make_shared(attributes.attn_scale_value.value()); + } + + // If attn scale present, add a pointwise mul node + if (attributes.inputs[input_names::Attn_scale]) { + Pointwise_attributes scale_attributes; + scale_attributes.set_name("attn_scale").set_mode(PointwiseMode_t::MUL); + auto const& attn_scale_output = + pointwise(last_output, attributes.inputs[input_names::Attn_scale], scale_attributes); + last_output = attn_scale_output; + } + + // Descale Q (only for non-MXFP8 per-tensor scaling) + // For MXFP8, descaling was done via block_scale_dequantize before matmul + if (!use_mxfp8 && attributes.inputs.find(input_names::Descale_Q) != attributes.inputs.end() && + attributes.inputs.at(input_names::Descale_Q) != nullptr) { + auto descale_q_attributes = Pointwise_attributes().set_mode(PointwiseMode_t::MUL).set_name("descale_q"); + last_output = pointwise(last_output, attributes.inputs.at(input_names::Descale_Q), descale_q_attributes); + } + + // Descale K (only for non-MXFP8 per-tensor scaling) + // For MXFP8, descaling was done via block_scale_dequantize before matmul + if (!use_mxfp8 && attributes.inputs.find(input_names::Descale_K) != attributes.inputs.end() && + attributes.inputs.at(input_names::Descale_K) != nullptr) { + auto descale_k_attributes = Pointwise_attributes().set_mode(PointwiseMode_t::MUL).set_name("descale_k"); + last_output = pointwise(last_output, attributes.inputs.at(input_names::Descale_K), descale_k_attributes); + } + + if (attributes.attention_score_modifier != nullptr) { + auto graph_ = std::make_shared(); + std::shared_ptr node_ = std::static_pointer_cast(graph_); + node_->context = this->context; + last_output = attributes.attention_score_modifier(graph_, last_output); + sub_nodes.emplace_back(node_); + } + + // Optional bias + if (attributes.inputs.find(input_names::Bias) != attributes.inputs.end() && + attributes.inputs[input_names::Bias]) { + auto graph_ = std::make_shared(); + std::shared_ptr node_ = std::static_pointer_cast(graph_); + node_->context = this->context; + last_output = attn::score_modifiers::bias(graph_, last_output, attributes.inputs[input_names::Bias]); + sub_nodes.emplace_back(node_); + } + + if (attributes.alibi_mask) { + auto graph_ = std::make_shared(); + std::shared_ptr node_ = std::static_pointer_cast(graph_); + node_->context = this->context; + last_output = attn::score_modifiers::alibi_mask(graph_, last_output, alibi_slopes, h_q, alibi_slopes_size); + sub_nodes.emplace_back(node_); + } + + // There are two cases of applying padding mask + // 1. when actual seq_len is less than or equal to max_seq_len + if (attributes.padding_mask) { + auto graph_ = std::make_shared(); + std::shared_ptr node_ = std::static_pointer_cast(graph_); + node_->context = this->context; + last_output = attn::score_modifiers::padding_mask(graph_, + last_output, + attributes.inputs[input_names::SEQ_LEN_KV], + attributes.inputs[input_names::SEQ_LEN_Q]); + sub_nodes.emplace_back(node_); + } + + // 2. (bug in cudnn backend) no padding with max_seq_len%64!=0 + if ((s_kv % 64 != 0) && (!(attributes.padding_mask)) && (detail::get_backend_version() < 90000)) { + auto col_index_attributes = + Pointwise_attributes().set_name("gen_col_index").set_mode(PointwiseMode_t::GEN_INDEX).set_axis(3); + auto col_index_output = pointwise(last_output, col_index_attributes); + // scalar seq_kv only needs to be passed in case there in no padding mask and seq_kv is not multiple of 64. + // Also future versions of cudnn will not need it, hence tensor is pre-fixed. + auto scalar_max_seq_kv = std::make_shared(static_cast(s_kv)); + + auto col_less_seq_kv_attributes = + Pointwise_attributes().set_name("col_less_seq_kv").set_mode(PointwiseMode_t::CMP_LT); + auto col_less_seq_kv_output = pointwise(col_index_output, scalar_max_seq_kv, col_less_seq_kv_attributes); + + // Lower attributes to binary select attributes + auto negative_inf_padding = + std::make_shared(attn::score_modifiers::get_negative_inf_value()); + auto binary_select_attributes = + Pointwise_attributes().set_name("binary_select").set_mode(PointwiseMode_t::BINARY_SELECT); + auto padding_mask_output = + pointwise(last_output, negative_inf_padding, col_less_seq_kv_output, binary_select_attributes); + last_output = padding_mask_output; + } + + // Apply (bottom-right) causal masking (with right bound) and/or set the left bound + if (attributes.left_bound.has_value() || attributes.right_bound.has_value()) { + auto graph_ = std::make_shared(); + std::shared_ptr node_ = std::static_pointer_cast(graph_); + node_->context = this->context; + + auto s_kv_ptr = attributes.inputs.find(input_names::SEQ_LEN_KV) != attributes.inputs.end() + ? attributes.inputs[input_names::SEQ_LEN_KV] + : nullptr; + auto s_q_ptr = attributes.inputs.find(input_names::SEQ_LEN_Q) != attributes.inputs.end() + ? attributes.inputs[input_names::SEQ_LEN_Q] + : nullptr; + + last_output = attn::score_modifiers::sliding_window_mask(graph_, + last_output, + attributes.diagonal_alignment, + attributes.left_bound, + attributes.right_bound, + s_q, + s_kv, + s_q_ptr, + s_kv_ptr, + /*cu_s_q_ptr=*/nullptr, + /*cu_s_kv_ptr=*/nullptr); + sub_nodes.emplace_back(node_); + } + + // Lower attributes to softmax attributes + auto softmax_output = std::make_shared(); + softmax_output->set_is_virtual(true); + + auto softmax_attributes = Softmax_attributes().set_name("softmax"); + // Set sink for softmax if user has provided a sink tensor + if (attributes.inputs.find(input_names::SINK_TOKEN) != attributes.inputs.end()) { + softmax_attributes.set_sink(attributes.inputs[input_names::SINK_TOKEN]); + } + // Special non-functional-style call. Needed because output already created and provided to user. + softmax(last_output, + softmax_attributes, + softmax_output, + attributes.outputs[output_names::Stats], + attributes.outputs[output_names::Max], + attributes.outputs[output_names::Sum_exp]); + last_output = softmax_output; + + // Two cases for training: dropout present or not + bool dropout_present = false; + auto const& dropout_mask = attributes.inputs.find(input_names::Dropout_mask); + bool const is_dropout_custom = (dropout_mask != attributes.inputs.end()) && (dropout_mask->second != nullptr); + if (attributes.dropout_probability.has_value()) { + dropout_present = true; + // Special case: Skip dropout when 0.0 probability. Only do for 8.9.3 and up as rng isn't optional earlier. + if (detail::get_backend_version() > 8902 && attributes.dropout_probability.value() == 0.0) { + dropout_present = false; + } + } else if (is_dropout_custom) { + dropout_present = true; + } + + if (dropout_present) { + if (is_dropout_custom) { + auto dropout_scale_attributes = + Pointwise_attributes().set_name("dropout_scale_mul").set_mode(PointwiseMode_t::MUL); + auto const& dropout_scale_output = + pointwise(last_output, attributes.inputs[input_names::Dropout_scale], dropout_scale_attributes); + + auto mask_attributes = + Pointwise_attributes().set_name("dropout_mask_mul").set_mode(PointwiseMode_t::MUL); + auto const& dropout_mask_output = + pointwise(dropout_scale_output, dropout_mask->second, mask_attributes); + last_output = dropout_mask_output; + } else { + if (attributes.outputs[output_names::RNG_DUMP] != nullptr) { + rng_output = attributes.outputs[output_names::RNG_DUMP]; + rng(attributes.inputs[input_names::Seed], + attributes.inputs[input_names::Offset], + Rng_attributes() + .set_name("rng") + .set_distribution(RngDistribution_t::BERNOULLI) + .set_bernoulli_probability(1.0 - attributes.dropout_probability.value()), + rng_output); + } else { + rng_output = rng(attributes.inputs[input_names::Seed], + attributes.inputs[input_names::Offset], + Rng_attributes() + .set_name("rng") + .set_distribution(RngDistribution_t::BERNOULLI) + .set_bernoulli_probability(1.0 - attributes.dropout_probability.value())); + rng_output + // Hard coding dim and strides as rng output can no inputs to infer it from. + ->set_dim({b, h_q, s_q, s_kv}) + .set_stride({h_q * s_q * s_kv, s_q * s_kv, s_kv, 1}); + } + + auto mask_attributes = + Pointwise_attributes().set_name("dropout_mask_mul").set_mode(PointwiseMode_t::MUL); + auto const& dropout_mask_output = pointwise(last_output, rng_output, mask_attributes); + last_output = dropout_mask_output; + + std::shared_ptr dropout_scale = nullptr; + + if (detail::get_backend_version() < 8903) { + half dropout_scale_value = __float2half(1.0f / (1.0f - attributes.dropout_probability.value())); + dropout_scale = std::make_shared(dropout_scale_value); + } else { + float dropout_scale_value = (1.0f / (1.0f - attributes.dropout_probability.value())); + dropout_scale = std::make_shared(dropout_scale_value); + } + + auto dropout_scale_attributes = + Pointwise_attributes().set_name("dropout_scale").set_mode(PointwiseMode_t::MUL); + auto const& dropout_scale_output = pointwise(last_output, dropout_scale, dropout_scale_attributes); + last_output = dropout_scale_output; + } + } + + // Amax S + if (attributes.outputs.find(output_names::Amax_S) != attributes.outputs.end() && + attributes.outputs.at(output_names::Amax_S) != nullptr) { + auto amax_attributes = Reduction_attributes().set_name("amax_s").set_mode(ReductionMode_t::AMAX); + // Special non-functional-style call. Needed because output already created and provided to user. + reduction(last_output, amax_attributes, attributes.outputs.at(output_names::Amax_S)); + } + + // Scale S + if (attributes.inputs.find(input_names::Scale_S) != attributes.inputs.end() && + attributes.inputs.at(input_names::Scale_S) != nullptr) { + auto scale_s_attributes = Pointwise_attributes().set_name("scale_s").set_mode(PointwiseMode_t::MUL); + last_output = pointwise(last_output, attributes.inputs.at(input_names::Scale_S), scale_s_attributes); + } + + // Lower attributes to bmm2 attributes + // Requirement by cudnn backend to take in bmm2 aType as i/o type. + last_output->set_data_type(attributes.inputs[input_names::Q]->get_data_type()); + + auto const& seq_len_q = attributes.inputs[input_names::SEQ_LEN_Q]; + auto const& seq_len_kv = attributes.inputs[input_names::SEQ_LEN_KV]; + // auto const& V = attributes.inputs[input_names::V]; + auto const& O = attributes.outputs[output_names::O]; + + std::shared_ptr v_cache; + + if (!is_paged_v()) { + v_cache = attributes.inputs[input_names::V]; + } else { + auto paged_cache_load_attributes_v = PagedCacheLoad_attributes().set_name("paged_v_cache_operation"); + v_cache = std::make_shared(); + v_cache->set_dim({b, h_v, s_kv, d_v}) + .set_stride({d_v * s_kv * h_v, d_v * s_kv, d_v, 1}) + .set_data_type(attributes.inputs[input_names::V]->get_data_type()); + v_cache->set_is_virtual(true); + paged_cache_load(attributes.inputs[input_names::V], + attributes.inputs[input_names::SEQ_LEN_KV], + attributes.inputs[input_names::Page_table_V], + paged_cache_load_attributes_v, + v_cache); + } + + //// S * V + if (attributes.mma_core_mode == DataType_t::HALF) { + auto bmm2_attributes = + Matmul_attributes().set_name("bmm2").set_m_override(seq_len_q).set_k_override(seq_len_kv); + // Special non-functional-style call. Needed because output already created and provided to user. + matmul(last_output, v_cache, bmm2_attributes, O); + } else if (attributes.mma_core_mode == DataType_t::FP8_E4M3 || + attributes.mma_core_mode == DataType_t::FP8_E5M2) { + if (use_mxfp8) { + // MXFP8: Dequantize V using block scale factors + // SF_V should be provided with dims [b, h, s_scale, d] where s is scaled + // (different from SF_Q/SF_K which have d scaled) + // No transformation needed - use SF_V as provided + + // Dequantize V with SF_V + auto dequant_v_attrs = Block_scale_dequantize_attributes() + .set_name("dequant_v") + .set_block_size({1, 32}); // Standard MXFP8 block size + auto v_dequant = std::make_shared(); + v_dequant->set_is_virtual(true); + v_dequant->set_dim(v_cache->get_dim()); + v_dequant->set_stride(v_cache->get_stride()); + block_scale_dequantize(v_cache, attributes.inputs[input_names::Descale_V], dequant_v_attrs, v_dequant); + + // Use regular matmul with dequantized inputs + auto bmm2_attributes = + Matmul_attributes().set_name("bmm2").set_m_override(seq_len_q).set_k_override(seq_len_kv); + // Special non-functional-style call. Needed because output already created and provided to user. + matmul(last_output, v_dequant, bmm2_attributes, O); + + // Compute Amax_O for MXFP8 + auto const& amax_o = attributes.outputs.at(output_names::Amax_O); + if (amax_o != nullptr) { + auto amax_attributes = Reduction_attributes().set_name("amax_o").set_mode(ReductionMode_t::AMAX); + // Special non-functional-style call. Needed because output already created and provided to user. + reduction(O, amax_attributes, amax_o); + } + } else { + // Regular per-tensor FP8 scaling + auto const& descale_s = attributes.inputs.at(input_names::Descale_S); + auto const& descale_v = attributes.inputs.at(input_names::Descale_V); + auto const& scale_o = attributes.inputs.at(input_names::Scale_O); + auto const& amax_o = attributes.outputs.at(output_names::Amax_O); + + auto bmm2_attributes = + Matmul_fp8_attributes().set_name("bmm2").set_m_override(seq_len_q).set_k_override(seq_len_kv); + // Special non-functional-style call. Needed because output already created and provided to user. + matmul_fp8(last_output, v_cache, descale_s, descale_v, scale_o, bmm2_attributes, O, amax_o); + } + } else { + RETURN_CUDNN_FRONTEND_ERROR_IF(true, error_code_t::GRAPH_NOT_SUPPORTED, "Unsupported MMA core mode"); + } + + return {error_code_t::OK, ""}; + } +}; + +class CompositeSDPABackwardNode : public NodeCRTP { + using input_names = SDPA_backward_attributes::input_names; + using output_names = SDPA_backward_attributes::output_names; + + private: + // non-virtual node gpu tensors + std::shared_ptr dQ_accum; + int64_t dQ_accum_size = 0; + std::shared_ptr dK_fullhead; + int64_t dK_fullhead_size = 0; + std::shared_ptr dV_fullhead; + int64_t dV_fullhead_size = 0; + std::shared_ptr softmax_sum; + int64_t softmax_sum_size = 0; + std::shared_ptr alibi_slopes; + int64_t alibi_slopes_size = 0; + + mutable bool has_workaround_padding_mask = false; // Will be edited in pre_validate_node() + mutable int32_t s_q_for_workaround_padding_mask = 0; // Will be edited in pre_validate_node() + mutable int32_t s_kv_for_workaround_padding_mask = 0; // Will be edited in pre_validate_node() + mutable std::shared_ptr + workaround_padding_mask_seq_len_q; // Will be edited in pre_validate_node() + mutable std::shared_ptr + workaround_padding_mask_seq_len_kv; // Will be edited in pre_validate_node() + mutable int64_t batch_size_for_workaround_padding_mask = 0; // Will be edited in pre_validate_node() + mutable bool is_deterministic_algorithm_supported_on_blackwell = false; // Will be edited in pre_validate_node() + mutable bool is_d256_on_blackwell = false; // Will be edited in pre_validate_node() + + // Promote any 1-D seq_len / ragged-offset index tensors to the 4-D + // [n, 1, 1, 1] form the cuDNN backend requires (see promote_1d_index_tensor_to_4d). + void + promote_index_tensors_to_4d() { + // TODO: Handle CU_SEQ_LEN_Q and CU_SEQ_LEN_KV once bprop supports these. + for (auto& key : {input_names::SEQ_LEN_Q, input_names::SEQ_LEN_KV}) { + auto const it = attributes.inputs.find(key); + if (it != attributes.inputs.end()) { + promote_1d_index_tensor_to_4d(it->second); + } + } + for (auto& [key, value] : attributes.inputs) { + CUDNN_FRONTEND_UNUSED(key); + if (value != nullptr) { + promote_1d_index_tensor_to_4d(value->get_ragged_offset()); + } + } + for (auto& [key, value] : attributes.outputs) { + CUDNN_FRONTEND_UNUSED(key); + if (value != nullptr) { + promote_1d_index_tensor_to_4d(value->get_ragged_offset()); + } + } + } + + public: + mutable SDPA_backward_attributes attributes; // Will be edited in pre_validate_node() for workaround padding mask + + CompositeSDPABackwardNode(SDPA_backward_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::COMPOSITE; + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating CompositeSDPABackwardNode" << attributes.name); + + // check that Q, K, V, O, stats, dO, dQ, dK, dV tensors has been assigned + // check that dim and strides has been assigned and last stride is 1 +#define CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(port, port_map) \ + { \ + std::shared_ptr tensor_ptr = port_map.at(port); \ + RETURN_CUDNN_FRONTEND_ERROR_IF(tensor_ptr->get_dim().size() != 4, \ + error_code_t::ATTRIBUTE_NOT_SET, \ + "The dim for " + std::string(#port) + " is invalid"); \ + RETURN_CUDNN_FRONTEND_ERROR_IF(tensor_ptr->get_stride().size() != 4, \ + error_code_t::ATTRIBUTE_NOT_SET, \ + "The stride for " + std::string(#port) + " is invalid"); \ + RETURN_CUDNN_FRONTEND_ERROR_IF( \ + tensor_ptr->get_stride()[3] != 1, \ + error_code_t::GRAPH_NOT_SUPPORTED, \ + "The stride for the last dimension corresponding to the embedding size per head should be 1 for " + \ + std::string(#port)); \ + } + + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::Q, attributes.inputs); + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::K, attributes.inputs); + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::V, attributes.inputs); + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::O, attributes.inputs); + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::Stats, attributes.inputs); + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::dO, attributes.inputs); + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(output_names::dQ, attributes.outputs); + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(output_names::dK, attributes.outputs); + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(output_names::dV, attributes.outputs); + +#undef CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE + + // validate backend limitations for the operation + // clang-format off + int64_t s_q = attributes.inputs.at(input_names::Q)->get_dim()[2]; + int64_t s_kv = attributes.inputs.at(input_names::V)->get_dim()[2]; + int64_t h_q = attributes.inputs.at(input_names::Q)->get_dim()[1]; + int64_t h_k = attributes.inputs.at(input_names::K)->get_dim()[1]; + int64_t h_v = attributes.inputs.at(input_names::V)->get_dim()[1]; + int64_t d_qk = attributes.inputs.at(input_names::Q)->get_dim()[3]; + int64_t d_v = attributes.inputs.at(input_names::V)->get_dim()[3]; + + bool const is_ragged = attributes.inputs.at(input_names::Q)->get_ragged_offset() || + attributes.inputs.at(input_names::K)->get_ragged_offset() || + attributes.inputs.at(input_names::V)->get_ragged_offset() || + attributes.inputs.at(input_names::O)->get_ragged_offset(); + + auto const& bias_mask = attributes.inputs.find(input_names::Bias); + bool const is_bias = (bias_mask != attributes.inputs.end() && bias_mask->second != nullptr); + auto const& dbias_mask = attributes.outputs.find(output_names::dBias); + bool const is_dbias = (dbias_mask != attributes.outputs.end() && dbias_mask->second != nullptr); + + auto const& dropout_mask = attributes.inputs.find(input_names::Dropout_mask); + bool const is_dropout_custom = (dropout_mask != attributes.inputs.end()) && (dropout_mask->second != nullptr); + bool const is_dropout = attributes.dropout_probability.has_value() || is_dropout_custom; + + auto const& rng_tensor = attributes.outputs.find(output_names::RNG_DUMP); + bool const is_rng = (rng_tensor != attributes.outputs.end() && rng_tensor->second != nullptr); + + // validation TODO: + // - validate stats has valid dims + // - validate Q and dQ have the same dims + + // Stop s_q = S_kv = 1 from running + RETURN_CUDNN_FRONTEND_ERROR_IF(s_q == 1 && s_kv == 1, + error_code_t::GRAPH_NOT_SUPPORTED, + "s_q = s_kv = 1 is not supported."); + + // Bug workarounds for known problematic versions (TE constraint) + RETURN_CUDNN_FRONTEND_ERROR_IF( + detail::get_backend_version() == 91000 || detail::get_backend_version() == 91001, + error_code_t::GRAPH_NOT_SUPPORTED, + "SDPA FP16/BF16 backward is not supported on cuDNN 9.10.0/9.10.1 due to known bugs. " + "Please consider upgrading to 9.10.2 or newer."); + + // 9.14.0 sliding window bug: non-causal + s_kv > 1024 + sliding window + RETURN_CUDNN_FRONTEND_ERROR_IF( + detail::get_backend_version() == 91400 && s_kv > 1024 && attributes.left_bound.has_value() && + !attributes.has_causal_like_masking(), + error_code_t::GRAPH_NOT_SUPPORTED, + "cuDNN 9.14.0 has a known bug with non-causal + s_kv > 1024 + sliding window attention. " + "Please consider upgrading to 9.14.1 or newer."); + + CHECK_CUDNN_FRONTEND_ERROR(context.populate_sm_version_from_device()); + int32_t const sm_version = context.get_sm_version(); + int32_t const prop_major = sm_version / 10; + + if (prop_major == 9) { + // validate basic dimension requirements + + if ((detail::get_backend_version() >= 91100) && (detail::get_backend_version() < 91300)) { + + if ((128 < d_qk) && (d_qk <= 192) && (64 < d_v) && (d_v <= 128)) { + + // DeepSeek case, 9.11 only supports 192 hidden dim + RETURN_CUDNN_FRONTEND_ERROR_IF( (d_v != 128) && (d_qk != 192), + error_code_t::GRAPH_NOT_SUPPORTED, + "Num hidden_dim d_v should be equal to 128 if d_qk is 192"); + } + } + + RETURN_CUDNN_FRONTEND_ERROR_IF((d_qk > 256) || (d_qk % 8 != 0) || (d_v > 256) || (d_v % 8 != 0), + error_code_t::GRAPH_NOT_SUPPORTED, + "Num hidden_dim should be less than or equal to 256 and hidden_dim should be multiple of 8"); + + } else if (prop_major == 10 && detail::get_backend_version() >= 91100) { + // validate basic dimension requirements + if (d_qk == 192) { // special case for 192 hidden dim + RETURN_CUDNN_FRONTEND_ERROR_IF( (d_v != 128), + error_code_t::GRAPH_NOT_SUPPORTED, + "Num hidden_dim d_v should be equal to 128 if d_qk is 192"); + } else if (detail::get_backend_version() >= 92300 && d_qk == 256 && d_v == 256) { + is_d256_on_blackwell = true; + attributes.is_deterministic_algorithm = true; + } else { + RETURN_CUDNN_FRONTEND_ERROR_IF((d_qk > 128) || (d_qk % 8 != 0) || (d_v > 128) || (d_v % 8 != 0), + error_code_t::GRAPH_NOT_SUPPORTED, + "Num hidden_dim should be less than or equal to 128 and hidden_dim should be multiple of 8 when d_qk != d_v"); + } + } else { + // validate basic dimension requirements + RETURN_CUDNN_FRONTEND_ERROR_IF((d_qk > 128) || (d_qk % 8 != 0) || (d_v > 128) || (d_v % 8 != 0), + error_code_t::GRAPH_NOT_SUPPORTED, + "Num hidden_dim should be less than or equal to 128 and hidden_dim should be multiple of 8"); + } + + RETURN_CUDNN_FRONTEND_ERROR_IF((attributes.attention_score_modifier != nullptr) && + (attributes.alibi_mask || attributes.padding_mask || attributes.has_causal_like_masking() || + attributes.left_bound.has_value()), error_code_t::GRAPH_NOT_SUPPORTED,"Attention score mod enabled and hence other subgraphs are disabled."); + + RETURN_CUDNN_FRONTEND_ERROR_IF((h_q % h_k != 0) || (h_q % h_v != 0), + error_code_t::GRAPH_NOT_SUPPORTED, + "For group-query attention, number of heads for key and query must be a factor of number of heads for query"); + + // validate options for attn_scale + auto const& attn_scale = attributes.inputs.find(input_names::Attn_scale); + bool const has_attn_scale = (attn_scale != attributes.inputs.end()) && (attn_scale->second != nullptr); + RETURN_CUDNN_FRONTEND_ERROR_IF(has_attn_scale && attributes.attn_scale_value.has_value(), + error_code_t::ATTRIBUTE_NOT_SET, + "attn_scale with tensor and value cannot be set at the same time."); + + // validate alibi requirements + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.alibi_mask && !(attributes.right_bound.has_value() && attributes.right_bound.value() == 0), + error_code_t::GRAPH_NOT_SUPPORTED, + "When alibi mask is used, diagonal_band_right_bound needs to be set to 0."); + + // validate options for bias mask + RETURN_CUDNN_FRONTEND_ERROR_IF(is_bias && (bias_mask->second->get_data_type() == DataType_t::BOOLEAN), + error_code_t::GRAPH_NOT_SUPPORTED, + "Bias mask data type cannot be boolean"); + + if (s_kv % 128 != 0 && attributes.padding_mask == false && is_ragged == false && detail::get_backend_version() <= 91500) { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Workaround padding mask is enabled for s_q % 128 != 0 and use_padding_mask == false and is_ragged == false"); + has_workaround_padding_mask = true; + batch_size_for_workaround_padding_mask = attributes.inputs.at(input_names::Q)->get_dim()[0]; + s_q_for_workaround_padding_mask = s_q; + s_kv_for_workaround_padding_mask = s_kv; + workaround_padding_mask_seq_len_q = std::make_shared(); + workaround_padding_mask_seq_len_q->set_name("workaround_padding_mask_seq_len_q").set_dim({batch_size_for_workaround_padding_mask,1,1,1}).set_stride({1,1,1,1}).set_data_type(DataType_t::INT32); + workaround_padding_mask_seq_len_kv = std::make_shared(); + workaround_padding_mask_seq_len_kv->set_name("workaround_padding_mask_seq_len_kv").set_dim({batch_size_for_workaround_padding_mask,1,1,1}).set_stride({1,1,1,1}).set_data_type(DataType_t::INT32); + attributes.set_padding_mask(true); + attributes.set_seq_len_q(workaround_padding_mask_seq_len_q).set_seq_len_kv(workaround_padding_mask_seq_len_kv); + } + + // validate options for padding mask + auto const& seq_len_q = attributes.inputs.find(input_names::SEQ_LEN_Q); + bool const has_seq_len_q = (seq_len_q != attributes.inputs.end()) && (seq_len_q->second != nullptr); + auto const& seq_len_kv = attributes.inputs.find(input_names::SEQ_LEN_KV); + bool const has_seq_len_kv = (seq_len_kv != attributes.inputs.end()) && (seq_len_kv->second != nullptr); + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.padding_mask && (!has_seq_len_q || !has_seq_len_kv), + error_code_t::ATTRIBUTE_NOT_SET, + "Padding mask requires seq_len_q and seq_len_kv to be set."); + RETURN_CUDNN_FRONTEND_ERROR_IF((!attributes.padding_mask && !attributes.attention_score_modifier) && (has_seq_len_q || has_seq_len_kv), + error_code_t::ATTRIBUTE_NOT_SET, + "seq_len_q and seq_len_kv needs to be set only if padding mask is enabled."); + + // validate options for max_total_seq_len + RETURN_CUDNN_FRONTEND_ERROR_IF((attributes.max_total_seq_len_q.has_value() || attributes.max_total_seq_len_kv.has_value()) && !is_ragged, + error_code_t::GRAPH_NOT_SUPPORTED, + "max_total_seq_len_q is only supported with packed layout"); + + // validate options for bottom right causal mask + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.has_causal_mask_bottom_right() && (!attributes.padding_mask) && s_q > s_kv, + error_code_t::GRAPH_NOT_SUPPORTED, + "Bottom right causal mask does not support max_s_q > max_s_kv. Please virtually slice the Q tensor and pass it as max_s_q == max_s_kv"); + + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.has_causal_mask_bottom_right() && (is_bias || attributes.alibi_mask || (is_ragged && !attributes.padding_mask) || is_dropout), + error_code_t::GRAPH_NOT_SUPPORTED, + "Bottom right causal mask is only supported with is_bias=False, is_alibi=False, is_dropout=False. Further is_ragged==True is only allowed when padding_mask=True."); + + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.has_causal_mask_bottom_right() && (detail::get_backend_version() < 90600) && ((s_q % 64 != 0) || (s_kv % 64 != 0)), + error_code_t::GRAPH_NOT_SUPPORTED, + "Bottom right causal mask is only supported with s_q multiple of 64, and s_kv multiple of 64, for cudnn version below 9.6.0"); + + // validate options for sliding window length + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.left_bound.has_value() && attributes.left_bound.value() <= 0, + error_code_t::INVALID_VALUE, + "Left bound (Sliding window length) should be greater than or equals to zero when set."); + + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.left_bound.has_value() && (s_q * attributes.left_bound.value() == s_kv * attributes.left_bound.value()) && (detail::get_backend_version() <= 90900) && (prop_major == 9) && attributes.has_causal_mask_bottom_right(), + error_code_t::GRAPH_NOT_SUPPORTED, + "On Hopper architecture, this specific combination of s_q, s_kv, and left_bound + right_bound + bottom right diagonal alignment is not supported for backend version 9.9 or below"); + + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.left_bound.has_value() && (!attributes.padding_mask) && s_q > s_kv, + error_code_t::GRAPH_NOT_SUPPORTED, + "Sliding window attention is only supported with max_s_q <= max_s_kv."); + + if ((detail::get_backend_version() >= 91002)) { + RETURN_CUDNN_FRONTEND_ERROR_IF((attributes.left_bound.has_value() || attributes.right_bound.has_value()) && ((is_ragged && !attributes.padding_mask)), + error_code_t::GRAPH_NOT_SUPPORTED, + "Left and right bounds with is_ragged==True is only allowed when padding_mask=True. And the diagonal alignment must be set."); + } else { + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.left_bound.has_value() && (! attributes.has_causal_like_masking() || is_dropout || is_bias || (is_ragged && !attributes.padding_mask)), + error_code_t::GRAPH_NOT_SUPPORTED, + "Left and right bounds are only supported with is_dropout=False, is_bias=False. Further is_ragged==True is only allowed when padding_mask=True. Lastly the diagonal alignment must be set."); + } + + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.right_bound.has_value() && attributes.right_bound.value() < 0, + error_code_t::INVALID_VALUE, + "Right bound needs to be larger than or equal to zero"); + + // validate options for dropout mask + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.dropout_probability.has_value() && is_dropout_custom, + error_code_t::ATTRIBUTE_NOT_SET, + "Using both, custom dropout mask and internal-mask generation using dropout probability, is ill-formed."); + + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.dropout_probability.has_value() && attributes.dropout_probability.value() == 1.0, + error_code_t::ATTRIBUTE_NOT_SET, + "Dropout probability cannot be 1 as corresponding scale wont be well formed."); + + // validate options for deterministic algorithm + if(attributes.is_deterministic_algorithm && (prop_major == 10)) { + RETURN_CUDNN_FRONTEND_ERROR_IF( (detail::get_backend_version() < 91800), + error_code_t::GRAPH_NOT_SUPPORTED, + "Deterministic algorithm is not supported on blackwell architecture with cudnn version below 9.18.0"); + + // dbias bias rng/dropout alibi + RETURN_CUDNN_FRONTEND_ERROR_IF(is_dbias || is_rng || is_dropout || attributes.alibi_mask, + error_code_t::GRAPH_NOT_SUPPORTED, + "Deterministic algorithm is not supported on blackwell architecture when dbias, rng/dropout, alibi is enabled"); + + is_deterministic_algorithm_supported_on_blackwell = true; + } + + if(detail::get_backend_version() >= 91801) { + RETURN_CUDNN_FRONTEND_ERROR_IF(is_ragged && (8 == prop_major || 12 == prop_major) && attributes.is_deterministic_algorithm, + error_code_t::GRAPH_NOT_SUPPORTED, + "Deterministic algorithm is not supported for bprop thd on SM8X and SM12X GPUs"); + + RETURN_CUDNN_FRONTEND_ERROR_IF(is_ragged && (8 == prop_major || 12 == prop_major) && attributes.inputs[input_names::Stats]->get_ragged_offset(), + error_code_t::GRAPH_NOT_SUPPORTED, + "Packed/ragged LSE is not supported for bprop thd on SM8X and SM12X GPUs"); + } + + // version specific validation + RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 90500 && is_dbias && attributes.padding_mask, + error_code_t::GRAPH_NOT_SUPPORTED, + "For cuDNN version below 9.5.0, dBias with variable sequence lengths is not supported"); + + RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 90500 && is_dbias && ((s_q % 64 != 0) || (s_kv % 64 != 0)), + error_code_t::GRAPH_NOT_SUPPORTED, + "For cuDNN version below 9.5.0, dBias not support s_q/s_kv which aren't multiple of 64"); + + RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 90600 && is_ragged && ((h_q != h_k) || (h_q != h_v)), + error_code_t::GRAPH_NOT_SUPPORTED, + "For cuDNN version below 9.6.0, group-query attention with raggged offset is not supported"); + + // TODO add version check once fixed + RETURN_CUDNN_FRONTEND_ERROR_IF(prop_major == 10 && is_ragged && is_dbias, + error_code_t::GRAPH_NOT_SUPPORTED, + "dbias with ragged is not supported for SM Major version 10"); + + // validate that datatype is set for the graph + RETURN_CUDNN_FRONTEND_ERROR_IF(this->context.get_intermediate_data_type() == DataType_t::NOT_SET, + error_code_t::ATTRIBUTE_NOT_SET, + "Intermediate tensor data type needs to be set as internal tensors require it."); + // If dsink is set, sink also needs to be set + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.outputs.find(output_names::DSINK_TOKEN) != attributes.outputs.end() && attributes.inputs.find(input_names::SINK_TOKEN) == attributes.inputs.end(), + error_code_t::ATTRIBUTE_NOT_SET, + "If dsink is set, sink also needs to be set."); + // clang-format on + + return {error_code_t::OK, ""}; + } + + error_t + infer_properties_node() override final { + // If a 1-D length/offset index tensor [n] is supplied, promote it to the 4-D + // [n, 1, 1, 1] (contiguous) form the cuDNN backend requires. + promote_index_tensors_to_4d(); + + // clang-format off + if (detail::get_backend_version() < 90600 && (attributes.max_total_seq_len_q.has_value() || attributes.max_total_seq_len_kv.has_value())) { + CUDNN_FE_LOG_LABEL_ENDL("WARNING: sdpa_backward.attributes.max_total_seq_len has been set, but cuDNN version is below 9.6.0 does not support max_total_seq_len_q. The workspace memory size required to execute this graph may be unexpectedly large"); + attributes.max_total_seq_len_q.reset(); + attributes.max_total_seq_len_kv.reset(); + } + + // TODO add version check once fixed + int64_t d_qk = attributes.inputs.at(input_names::Q)->get_dim()[3]; + int64_t d_v = attributes.inputs.at(input_names::V)->get_dim()[3]; + if ((attributes.max_total_seq_len_q.has_value() || attributes.max_total_seq_len_kv.has_value()) && (d_qk % 16 != 0 || d_v % 16 != 0)) { + CUDNN_FE_LOG_LABEL_ENDL("WARNING: sdpa_backward.attributes.max_total_seq_len has been set, but d is not a multiple of 16 has a known functional issue. The workspace memory size required to execute this graph may be unexpectedly large"); + attributes.max_total_seq_len_q.reset(); + attributes.max_total_seq_len_kv.reset(); + } + + + if(detail::get_backend_version() >= 91801) { + int32_t const prop_major = context.get_sm_version() / 10; + if(8 == prop_major || 12 == prop_major) { + if(attributes.max_total_seq_len_q.has_value() || attributes.max_total_seq_len_kv.has_value()) { + attributes.max_total_seq_len_q.reset(); + attributes.max_total_seq_len_kv.reset(); + CUDNN_FE_LOG_LABEL_ENDL("WARNING: sdpa_backward.attributes.max_total_seq_len has been set, but ampere style kernels have a known functional issue. The workspace memory size required to execute this graph may be unexpectedly large"); + + } + } + } + // clang-format on + + return {error_code_t::OK, ""}; + } + + error_t + expand_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for CompositeSDPABackwardNode " << attributes.name); + + attributes.fill_from_context(context); + + // Gather dim to fill properties of virtual tensors + auto const& q_dim = attributes.inputs[input_names::Q]->get_dim(); + auto b = q_dim[0]; + auto h_q = q_dim[1]; + auto s_q = q_dim[2]; + auto d_qk = q_dim[3]; + auto const& k_dim = attributes.inputs[input_names::K]->get_dim(); + auto h_k = k_dim[1]; + auto s_kv = k_dim[2]; + auto const& v_dim = attributes.inputs[input_names::V]->get_dim(); + auto h_v = v_dim[1]; + auto d_v = v_dim[3]; + + // cuDNN frontend API attention requires Q, K, V where + // Q = {b, h_q, s_q, d_qk} + // K = {b, h_k, s_kv, d_qk} + // V = {b, h_v, s_kv, d_v} + // but cuDNN backend API attention requires Q, KT, VT + // Q = {b, h_q, s_q, d_qk} + // KT = {b, h_k, d_qk, s_kv} + // VT = {b, h_v, d_v, s_kv} + // So the code below maps the K->KT and V->VT + std::vector temp_vec; + + temp_vec = attributes.inputs[input_names::K]->get_dim(); + std::swap(temp_vec[2], temp_vec[3]); + attributes.inputs[input_names::K]->set_dim(temp_vec); + + temp_vec = attributes.inputs[input_names::K]->get_stride(); + std::swap(temp_vec[2], temp_vec[3]); + attributes.inputs[input_names::K]->set_stride(temp_vec); + + temp_vec = attributes.inputs[input_names::V]->get_dim(); + std::swap(temp_vec[2], temp_vec[3]); + attributes.inputs[input_names::V]->set_dim(temp_vec); + + temp_vec = attributes.inputs[input_names::V]->get_stride(); + std::swap(temp_vec[2], temp_vec[3]); + attributes.inputs[input_names::V]->set_stride(temp_vec); + + std::shared_ptr last_output, exp_s_output, dS_output, rng_output; + + // --------------Initialize and create tensors before creating nodes-------------------- + // one_tensor is needed for non-dropout graphs + // one_tensor is passed by the node + auto one_tensor = std::make_shared(1.0f); + + if (attributes.attn_scale_value.has_value()) { + attributes.inputs[input_names::Attn_scale] = + std::make_shared(attributes.attn_scale_value.value()); + } + + // if dropout_mask is used, then the user passes scale and scale_inverse + bool is_dropout_prob = (attributes.dropout_probability.has_value()); + bool is_dropout_mask = (attributes.inputs[input_names::Dropout_mask] != nullptr); + if (is_dropout_prob) { + float dropout_scale_value = 1.0f / (1.0f - attributes.dropout_probability.value()); + float dropout_scale_inv_value = (1.0f - attributes.dropout_probability.value()); + + attributes.inputs[input_names::Dropout_scale] = std::make_shared(dropout_scale_value); + attributes.inputs[input_names::Dropout_scale_inv] = + std::make_shared(dropout_scale_inv_value); + } + + // ---------------------input tensor workarounds--------------------------- + + bool use_dp_workspace = false; + + int32_t const prop_major = context.get_sm_version() / 10; + + if (detail::get_backend_version() >= 8905 && detail::get_backend_version() < 90000) { + // workspace optimization is enabled by default when: + // 8.9.5 <= cudnn version < 9.0.0 + // device >= hopper + // batch * num_heads * seq_len_q * seq_len_kv * 2 <= dP workspace limit + // + // This following environment variable allows you to control the dP workspace limit. + // From cuDNN version 9.0.0, this option is obsolete will be ignored. + // CUDNN_FRONTEND_ATTN_DP_WORKSPACE_LIMIT=unset - enable workspace opt. until the default 256MB limit. + // CUDNN_FRONTEND_ATTN_DP_WORKSPACE_LIMIT=-1 - always enable workspace opt. + // CUDNN_FRONTEND_ATTN_DP_WORKSPACE_LIMIT=0 - always disable workspace opt. + // CUDNN_FRONTEND_ATTN_DP_WORKSPACE_LIMIT=n - enable workspace opt. until the n byte limit + + // hopper or above + if (prop_major >= 9) { + // default upper limit for workspace 256MB + int64_t max_dp_workspace_bytes = 256 * 1024 * 1024; + + // allow setting the upper limit with envvars + const char* env_dp_workspace_limit_char = get_environment("CUDNN_FRONTEND_ATTN_DP_WORKSPACE_LIMIT"); + if (env_dp_workspace_limit_char) { + char* end_ptr = nullptr; + max_dp_workspace_bytes = std::strtoll(env_dp_workspace_limit_char, &end_ptr, 10); + + if (*end_ptr != '\0') { + RETURN_CUDNN_FRONTEND_ERROR_IF(true, + error_code_t::ATTRIBUTE_NOT_SET, + "Invalid argument for CUDNN_FRONTEND_ATTN_DP_WORKSPACE_LIMIT " + "(int64_t; in bytes)"); + } + } + + int64_t workspace_s_q = ((s_q + 64 - 1) / 64) * 64; + int64_t workspace_s_kv = ((s_kv + 64 - 1) / 64) * 64; + int64_t required_dp_workspace_bytes = b * h_q * workspace_s_q * workspace_s_kv * 2; + + if (max_dp_workspace_bytes == -1) { + use_dp_workspace = true; + } else if (max_dp_workspace_bytes == 0) { + use_dp_workspace = false; + } else { + use_dp_workspace = (required_dp_workspace_bytes <= max_dp_workspace_bytes); + } + } + } + + // Force dP workspace implementation if: + // - dBias is enabled (dBias is only supported on workspace implementation) + // - the user force requests deterministic algorithm on hopper + if (attributes.outputs[output_names::dBias] || attributes.is_deterministic_algorithm) { + use_dp_workspace = true; + } + + // --------------RNG node-------------------- + + if (is_dropout_prob) { + if (attributes.outputs[output_names::RNG_DUMP] != nullptr) { + rng_output = attributes.outputs[output_names::RNG_DUMP]; + rng(attributes.inputs[input_names::Seed], + attributes.inputs[input_names::Offset], + Rng_attributes() + .set_name("rng") + .set_distribution(RngDistribution_t::BERNOULLI) + .set_bernoulli_probability(1.0f - attributes.dropout_probability.value()), + rng_output); + } else { + rng_output = rng(attributes.inputs[input_names::Seed], + attributes.inputs[input_names::Offset], + Rng_attributes() + .set_name("rng") + .set_distribution(RngDistribution_t::BERNOULLI) + .set_bernoulli_probability(1.0f - attributes.dropout_probability.value())); + rng_output->set_dim({b, h_q, s_q, s_kv}).set_stride({h_q * s_q * s_kv, s_q * s_kv, s_kv, 1}); + } + } else if (is_dropout_mask) { + rng_output = attributes.inputs[input_names::Dropout_mask]; + } + + // --------------"dO * o => softmax_sum" chain-------------------- + + // last_output = dO * O + last_output = pointwise(attributes.inputs[input_names::dO], + attributes.inputs[input_names::O], + Pointwise_attributes().set_name("mul_dO_O").set_mode(PointwiseMode_t::MUL)); + last_output->set_dim({b, h_q, s_q, d_v}).set_stride({h_q * s_q * d_v, s_q * d_v, h_q * d_v, 1}); + + // last_output = reduce(last_output, "b hq sq dv -> b hq sq 1") + last_output = + reduction(last_output, Reduction_attributes().set_name("reduce_dO_o").set_mode(ReductionMode_t::ADD)); + last_output->set_dim({b, h_q, s_q, 1}).set_stride({h_q * s_q, s_q, 1, 1}); + + if (attributes.outputs.find(output_names::DSINK_TOKEN) != attributes.outputs.end()) { + // sub_sink = sink - stats + auto sub_sink = pointwise(attributes.inputs[input_names::SINK_TOKEN], + attributes.inputs[input_names::Stats], + Pointwise_attributes().set_name("sub_sink").set_mode(PointwiseMode_t::SUB)); + + // exp_sink = exp(sub_sink) + auto exp_sink = + pointwise(sub_sink, Pointwise_attributes().set_name("exp_sink").set_mode(PointwiseMode_t::EXP)); + + // per_token_grad = exp_sink * last_output + auto per_token_grad = + pointwise(exp_sink, + last_output, + Pointwise_attributes().set_name("mul_exp_sink_last_output").set_mode(PointwiseMode_t::MUL)); + + // dSink = redduce(per_token_grad) + reduction(per_token_grad, + Reduction_attributes().set_name("reduce_per_token_grad").set_mode(ReductionMode_t::ADD), + attributes.outputs[output_names::DSINK_TOKEN]); + } + + // softmax_sum = last_output * dropout_scale + last_output = pointwise(last_output, + attributes.inputs[input_names::Dropout_scale_inv] + ? attributes.inputs[input_names::Dropout_scale_inv] + : one_tensor, + Pointwise_attributes().set_name("scale_dropout_inv").set_mode(PointwiseMode_t::MUL)); + last_output->set_dim({b, h_q, s_q, 1}).set_stride({h_q * s_q, s_q, 1, 1}); + + softmax_sum = last_output; + softmax_sum->set_is_virtual(false); + softmax_sum->set_dim({b, h_q, s_q, 1}); + softmax_sum->set_data_type(DataType_t::FLOAT); + + if (attributes.inputs[input_names::Stats]->get_ragged_offset() && attributes.max_total_seq_len_q.has_value()) { + // sized TH1 softmax_sum + softmax_sum->set_stride(attributes.inputs[input_names::Stats]->get_stride()); + softmax_sum->set_ragged_offset(attributes.inputs[input_names::Stats]->get_ragged_offset()); + softmax_sum_size = attributes.max_total_seq_len_q.value() * + (attributes.inputs[input_names::Stats]->get_stride())[2] * sizeof(float); + } else { + // sized BHS1 softmax_sum + softmax_sum->set_stride({h_q * s_q, s_q, 1, 1}); + softmax_sum_size = b * h_q * s_q * 1 * sizeof(float); + } + + // --------------"Q @ KT => exp_softmax => dV" chain-------------------- + + // s = einsum(q, k, "b hq sq dqk, b (hk g) skv dqk -> b hq sq skv", g=hq//hk) + last_output = matmul(attributes.inputs[input_names::Q], + attributes.inputs[input_names::K], + Matmul_attributes() + .set_name("matmul_Q_KT") + .set_m_override(attributes.inputs[input_names::SEQ_LEN_Q]) + .set_n_override(attributes.inputs[input_names::SEQ_LEN_KV])); + last_output->set_dim({b, h_q, s_q, s_kv}).set_stride({h_q * s_q * s_kv, s_q * s_kv, s_kv, 1}); + + // last_output = last_output * attention_scale + if (attributes.inputs[input_names::Attn_scale]) { + last_output = pointwise(last_output, + attributes.inputs[input_names::Attn_scale], + Pointwise_attributes().set_name("mul_s_attn_scale").set_mode(PointwiseMode_t::MUL)); + } + + if (attributes.attention_score_modifier != nullptr) { + auto graph_ = std::make_shared(); + std::shared_ptr node_ = std::static_pointer_cast(graph_); + node_->context = context; + last_output = attributes.attention_score_modifier(graph_, last_output); + sub_nodes.emplace_back(node_); + } + + // (optional) last_output = last_output + bias + if (attributes.inputs.find(input_names::Bias) != attributes.inputs.end() && + attributes.inputs[input_names::Bias]) { + auto graph_ = std::make_shared(); + std::shared_ptr node_ = std::static_pointer_cast(graph_); + node_->context = context; + last_output = attn::score_modifiers::bias(graph_, last_output, attributes.inputs[input_names::Bias]); + sub_nodes.emplace_back(node_); + } + + // (optional) last_output = last_output + alibi_mask + if (attributes.alibi_mask) { + auto graph_ = std::make_shared(); + std::shared_ptr node_ = std::static_pointer_cast(graph_); + node_->context = context; + last_output = attn::score_modifiers::alibi_mask(graph_, last_output, alibi_slopes, h_q, alibi_slopes_size); + sub_nodes.emplace_back(node_); + } + + // last_output = last_output - stats + last_output = pointwise(last_output, + attributes.inputs[input_names::Stats], + Pointwise_attributes().set_name("sub_s_m").set_mode(PointwiseMode_t::SUB)); + + // (optional) Apply padding mask + if (attributes.padding_mask) { + auto graph_ = std::make_shared(); + std::shared_ptr node_ = std::static_pointer_cast(graph_); + node_->context = context; + last_output = attn::score_modifiers::padding_mask(graph_, + last_output, + attributes.inputs[input_names::SEQ_LEN_KV], + attributes.inputs[input_names::SEQ_LEN_Q]); + sub_nodes.emplace_back(node_); + } + + // Explicitly put the padding value again after the stats have been loaded + if (attributes.padding_mask && detail::get_backend_version() >= 90000 && + detail::get_backend_version() < 91000) { + auto row_idx_output = pointwise(last_output, + Pointwise_attributes() + .set_name("gen_row_idx_2nd_padding") + .set_mode(PointwiseMode_t::GEN_INDEX) + .set_axis(2) + .set_compute_data_type(DataType_t::INT32)); + row_idx_output->set_data_type(DataType_t::INT32); + + auto col_idx_output = pointwise(last_output, + Pointwise_attributes() + .set_name("gen_col_idx_2nd_padding") + .set_mode(PointwiseMode_t::GEN_INDEX) + .set_axis(3) + .set_compute_data_type(DataType_t::INT32)); + col_idx_output->set_data_type(DataType_t::INT32); + + auto row_mask_output = pointwise(row_idx_output, + attributes.inputs[input_names::SEQ_LEN_Q], + Pointwise_attributes() + .set_name("lt_row_sq_2nd_padding") + .set_mode(PointwiseMode_t::CMP_LT) + .set_compute_data_type(DataType_t::BOOLEAN)); + row_mask_output->set_data_type(DataType_t::BOOLEAN); + + auto col_mask_output = pointwise(col_idx_output, + attributes.inputs[input_names::SEQ_LEN_KV], + Pointwise_attributes() + .set_name("lt_col_skv_2nd_padding") + .set_mode(PointwiseMode_t::CMP_LT) + .set_compute_data_type(DataType_t::BOOLEAN)); + col_mask_output->set_data_type(DataType_t::BOOLEAN); + + auto padding_mask_output = pointwise(row_mask_output, + col_mask_output, + Pointwise_attributes() + .set_name("and_row_col_2nd_padding") + .set_mode(PointwiseMode_t::LOGICAL_AND) + .set_compute_data_type(DataType_t::BOOLEAN)); + padding_mask_output->set_data_type(DataType_t::BOOLEAN); + auto negative_inf_padding = + std::make_shared(attn::score_modifiers::get_negative_inf_value()); + + last_output = pointwise( + last_output, + negative_inf_padding, + padding_mask_output, + Pointwise_attributes().set_name("select_2nd_padding").set_mode(PointwiseMode_t::BINARY_SELECT)); + } + + // Apply (bottom-right) causal masking (with right bound) and/or set the left bound + if (attributes.left_bound.has_value() || attributes.right_bound.has_value()) { + auto graph_ = std::make_shared(); + std::shared_ptr node_ = std::static_pointer_cast(graph_); + node_->context = context; + + auto s_kv_ptr = attributes.inputs.find(input_names::SEQ_LEN_KV) != attributes.inputs.end() + ? attributes.inputs[input_names::SEQ_LEN_KV] + : nullptr; + auto s_q_ptr = attributes.inputs.find(input_names::SEQ_LEN_Q) != attributes.inputs.end() + ? attributes.inputs[input_names::SEQ_LEN_Q] + : nullptr; + + last_output = attn::score_modifiers::sliding_window_mask(graph_, + last_output, + attributes.diagonal_alignment, + attributes.left_bound, + attributes.right_bound, + s_q, + s_kv, + s_q_ptr, + s_kv_ptr, + /*cu_s_q_ptr=*/nullptr, + /*cu_s_kv_ptr=*/nullptr); + sub_nodes.emplace_back(std::move(node_)); + } + + // last_output = exp(last_output) + last_output = pointwise(last_output, Pointwise_attributes().set_name("exp_s").set_mode(PointwiseMode_t::EXP)); + + exp_s_output = last_output; + + // (optional) last_output = last_output * dropout rng_output + if (is_dropout_prob || is_dropout_mask) { + last_output = + pointwise(last_output, + rng_output, + Pointwise_attributes().set_name("mul_p_dropout_mask").set_mode(PointwiseMode_t::MUL)); + } + + // (optional) last_output = last_output * dropout_scale + if (attributes.inputs[input_names::Dropout_scale]) { + last_output = + pointwise(last_output, + attributes.inputs[input_names::Dropout_scale], + Pointwise_attributes().set_name("mul_p_dropout_scale").set_mode(PointwiseMode_t::MUL)); + } + + // dV = einsum(p, dO, "b hq sq skv", "b hq sq dv -> b hq skv dv") + // if GQA, then dV = reduce(dV, "b (hv g) skv dv -> b hv skv dv", g=hq//hv) + // as reshape + matmul + last_output = reshape(last_output, Reshape_attributes().set_name("reshape_p")); + last_output->set_dim({b, h_q, s_kv, s_q}).set_stride({h_q * s_q * s_kv, s_q * s_kv, 1, s_kv}); + last_output->set_data_type(attributes.inputs[input_names::Q]->get_data_type()); + + if (h_q == h_v) { + // for MHA + matmul(last_output, + attributes.inputs[input_names::dO], + Matmul_attributes() + .set_name("matmul_pT_dO") + .set_m_override(attributes.inputs[input_names::SEQ_LEN_KV]) + .set_k_override(attributes.inputs[input_names::SEQ_LEN_Q]), + attributes.outputs[output_names::dV]); + } else { + // for GQA and MQA + dV_fullhead = matmul(last_output, + attributes.inputs[input_names::dO], + Matmul_attributes() + .set_name("matmul_pT_dO") + .set_m_override(attributes.inputs[input_names::SEQ_LEN_KV]) + .set_k_override(attributes.inputs[input_names::SEQ_LEN_Q])); + + dV_fullhead->set_dim({b, h_q, s_kv, d_v}); + dV_fullhead->set_data_type(attributes.inputs[input_names::Q]->get_data_type()); + + if (attributes.outputs[output_names::dV]->get_ragged_offset() && + attributes.max_total_seq_len_kv.has_value()) { + // map dV strides to dV_fullhead strides + std::vector dV_fullhead_stride = attributes.outputs[output_names::dV]->get_stride(); + dV_fullhead_stride[2] = dV_fullhead_stride[2] * (h_q / h_v); // sequence stride + dV_fullhead_stride[0] = dV_fullhead_stride[0] * (h_q / h_v); // batch stride + dV_fullhead->set_stride(dV_fullhead_stride); + // map dV ragged offset to dV_fullhead ragged offset with implicit multiplier + // implicit multiplier = h_q / h_v + dV_fullhead->set_ragged_offset(attributes.outputs[output_names::dV]->get_ragged_offset()); + // non virtual dV full head + dV_fullhead->set_is_virtual(false); + dV_fullhead_size = attributes.max_total_seq_len_kv.value() * dV_fullhead_stride[2] * sizeof(float); + } else { + // sized BHSD dQ_accum + dV_fullhead->set_stride({h_q * s_kv * d_v, s_kv * d_v, d_v, 1}); + } + + reduction(dV_fullhead, + Reduction_attributes().set_name("red_dV_head").set_mode(ReductionMode_t::ADD), + attributes.outputs[output_names::dV]); + } + + // --------------"dO @ VT => dS_output => dK" chain-------------------- + + // dP = einsum(dO, v, "b hq sq dv, b (hv g) skv dv -> b hq sq skv", g=hq//hv) + last_output = matmul(attributes.inputs[input_names::dO], + attributes.inputs[input_names::V], + Matmul_attributes() + .set_name("matmul_dO_VT") + .set_m_override(attributes.inputs[input_names::SEQ_LEN_Q]) + .set_n_override(attributes.inputs[input_names::SEQ_LEN_KV])); + last_output->set_dim({b, h_q, s_q, s_kv}).set_stride({h_q * s_q * s_kv, s_q * s_kv, s_kv, 1}); + + // last_output = last_output(dP) * mask + if (is_dropout_prob || is_dropout_mask) { + last_output = pointwise(last_output, + rng_output, + Pointwise_attributes().set_name("dP_dropout_mask").set_mode(PointwiseMode_t::MUL)); + } + + // last_output = last_output - softmax_sum + last_output = pointwise(last_output, + softmax_sum, + Pointwise_attributes().set_name("sub_dP_softmax_sum").set_mode(PointwiseMode_t::SUB)); + + // last_output = last_output * exp_s_output + last_output = pointwise( + last_output, exp_s_output, Pointwise_attributes().set_name("mul_dP_exp_s").set_mode(PointwiseMode_t::MUL)); + + // (optional) last_output = last_output * dropout_scale + if (attributes.inputs[input_names::Dropout_scale]) { + last_output = + pointwise(last_output, + attributes.inputs[input_names::Dropout_scale], + Pointwise_attributes().set_name("mul_dS_dropout_scale").set_mode(PointwiseMode_t::MUL)); + } + + if (attributes.outputs[output_names::dBias]) { + reduction(last_output, + Reduction_attributes().set_name("red_dP_dBias").set_mode(ReductionMode_t::ADD), + attributes.outputs[output_names::dBias]); + } + + // apply the bprop of attention score modifier + if (attributes.attention_score_modifier_bprop != nullptr) { + auto graph_ = std::make_shared(); + std::shared_ptr node_ = std::static_pointer_cast(graph_); + node_->context = context; + last_output = attributes.attention_score_modifier_bprop(graph_, last_output); + sub_nodes.emplace_back(node_); + } + + // (optional) last_output = last_output * bmm_scale + if (attributes.inputs[input_names::Attn_scale]) { + last_output = + pointwise(last_output, + attributes.inputs[input_names::Attn_scale], + Pointwise_attributes().set_name("mul_dS_attn_scale").set_mode(PointwiseMode_t::MUL)); + } + + dS_output = last_output; + + // dK = einsum(dS, Q, "b hq sq skv", "b hq sq dqk -> b hq skv dqk") + // if GQA, then dK = reduce(dK, "b (hk g) skv dqk -> b hk skv dqk", hq//hk) + // as reshape + matmul + last_output = reshape(last_output, Reshape_attributes().set_name("reshape_dS")); + last_output->set_dim({b, h_q, s_kv, s_q}).set_stride({h_q * s_q * s_kv, s_q * s_kv, 1, s_kv}); + last_output->set_data_type(attributes.inputs[input_names::Q]->get_data_type()); + + if (h_q == h_k) { + // for MHA + matmul(last_output, + attributes.inputs[input_names::Q], + Matmul_attributes() + .set_name("matmul_dST_Q") + .set_m_override(attributes.inputs[input_names::SEQ_LEN_KV]) + .set_k_override(attributes.inputs[input_names::SEQ_LEN_Q]), + attributes.outputs[output_names::dK]); + } else { + // for GQA and MQA + dK_fullhead = matmul(last_output, + attributes.inputs[input_names::Q], + Matmul_attributes() + .set_name("matmul_dST_Q") + .set_m_override(attributes.inputs[input_names::SEQ_LEN_KV]) + .set_k_override(attributes.inputs[input_names::SEQ_LEN_Q])); + + dK_fullhead->set_dim({b, h_q, s_kv, d_qk}); + dK_fullhead->set_data_type(attributes.inputs[input_names::Q]->get_data_type()); + + if (attributes.outputs[output_names::dK]->get_ragged_offset() && + attributes.max_total_seq_len_kv.has_value()) { + // sized THD dK_full_heads + // map dK strides to dK_fullhead strides + std::vector dK_fullhead_stride = attributes.outputs[output_names::dK]->get_stride(); + dK_fullhead_stride[0] = dK_fullhead_stride[0] * (h_q / h_k); // batch stride + dK_fullhead_stride[2] = dK_fullhead_stride[2] * (h_q / h_k); // sequence stride + dK_fullhead->set_stride(dK_fullhead_stride); + // map dK ragged offset to dK_fullhead ragged offset with implicit multiplier + // implicit multiplier = h_q / h_k + dK_fullhead->set_ragged_offset(attributes.outputs[output_names::dK]->get_ragged_offset()); + // non virtual dK full head + dK_fullhead->set_is_virtual(false); + dK_fullhead_size = attributes.max_total_seq_len_kv.value() * dK_fullhead_stride[2] * sizeof(float); + } else { + // sized BHSD dQ_accum + dK_fullhead->set_stride({h_q * s_kv * d_qk, s_kv * d_qk, d_qk, 1}); + } + + reduction(dK_fullhead, + Reduction_attributes().set_name("red_dK_head").set_mode(ReductionMode_t::ADD), + attributes.outputs[output_names::dK]); + } + + // --------------"dp_scaled @ K => dQ" chain-------------------- + + auto const& kt_dim = attributes.inputs[input_names::K]->get_dim(); + auto const& kt_stride = attributes.inputs[input_names::K]->get_stride(); + + // dQ = einsum(dS, K, "b hq sq skv, b (hk g) skv dqk -> b hq sq dqk", g=hq//hk) + // as reshape + matmul + last_output = reshape(attributes.inputs[input_names::K], Reshape_attributes().set_name("reshape_k")); + last_output->set_dim({kt_dim[0], kt_dim[1], kt_dim[3], kt_dim[2]}) + .set_stride({kt_stride[0], kt_stride[1], kt_stride[3], kt_stride[2]}); + + if (attributes.inputs[input_names::K]->get_ragged_offset() != nullptr) { + last_output->set_ragged_offset(attributes.inputs[input_names::K]->get_ragged_offset()); + } + + if (!use_dp_workspace) { + dQ_accum = std::make_shared(); + dQ_accum->set_is_virtual(false); + dQ_accum->set_dim({b, h_q, s_q, d_qk}); + dQ_accum->set_data_type(DataType_t::FLOAT); + + if (attributes.outputs[output_names::dQ]->get_ragged_offset() && + attributes.max_total_seq_len_q.has_value()) { + // sized THD dQ_accum + dQ_accum->set_stride(attributes.outputs[output_names::dQ]->get_stride()); + dQ_accum->set_ragged_offset(attributes.outputs[output_names::dQ]->get_ragged_offset()); + dQ_accum_size = attributes.max_total_seq_len_q.value() * + (attributes.outputs[output_names::dQ]->get_stride())[2] * sizeof(float); + } else { + // sized BHSD dQ_accum + dQ_accum->set_stride({h_q * s_q * d_qk, s_q * d_qk, d_qk, 1}); + dQ_accum_size = b * h_q * s_q * d_qk * sizeof(float); + } + + matmul(dS_output, + last_output, + Matmul_attributes() + .set_name("matmul_dS_K") + .set_m_override(attributes.inputs[input_names::SEQ_LEN_Q]) + .set_k_override(attributes.inputs[input_names::SEQ_LEN_KV]), + dQ_accum); + + pointwise(dQ_accum, + Pointwise_attributes().set_name("identity_dQ").set_mode(PointwiseMode_t::IDENTITY), + attributes.outputs[output_names::dQ]); + } else { + matmul(dS_output, + last_output, + Matmul_attributes() + .set_name("matmul_dS_K") + .set_m_override(attributes.inputs[input_names::SEQ_LEN_Q]) + .set_k_override(attributes.inputs[input_names::SEQ_LEN_KV]), + attributes.outputs[output_names::dQ]); + } + + return {error_code_t::OK, ""}; + } + + std::pair> + override_heuristics_query() const { + int32_t const sm_version = context.get_sm_version(); + bool const use_new_knobs = detail::get_backend_version() >= 92300; + // {128,128} bprop: tileM=3, tileN=2, kernelCfg=2(bprop warp), streamK=0, cgaM=0 + if (sm_version > 103 && (is_deterministic_algorithm_supported_on_blackwell)) { + if (use_new_knobs) { + return {17, + {{KnobType_t::TILE_M, 3}, + {KnobType_t::TILE_N, 2}, + {KnobType_t::KERNEL_CFG, 2}, + {KnobType_t::STREAM_K, 0}, + {KnobType_t::TILE_CGA_M, 0}, + {KnobType_t::STAGES, is_d256_on_blackwell ? 3 : 2}}}; + } else { + return {17, {{KnobType_t::KERNEL_CFG, 31}, {KnobType_t::STAGES, is_d256_on_blackwell ? 3 : 2}}}; + } + } else if (is_deterministic_algorithm_supported_on_blackwell) { + if (use_new_knobs) { + return {5, + {{KnobType_t::TILE_M, 3}, + {KnobType_t::TILE_N, 2}, + {KnobType_t::KERNEL_CFG, 2}, + {KnobType_t::STREAM_K, 0}, + {KnobType_t::TILE_CGA_M, 0}, + {KnobType_t::STAGES, is_d256_on_blackwell ? 3 : 2}}}; + } else { + return {5, {{KnobType_t::KERNEL_CFG, 31}, {KnobType_t::STAGES, is_d256_on_blackwell ? 3 : 2}}}; + } + } else { + return {-1, {}}; + } + } + + virtual int64_t + get_fe_workspace_size_node() const override final { + int64_t size = 0; + + size += ((alibi_slopes_size + 15) / 16 * 16); // align alibi slopes memory to 16 bytes + size += dQ_accum_size; + size += dK_fullhead_size; + size += dV_fullhead_size; + size += softmax_sum_size; + + if (has_workaround_padding_mask) { + size += batch_size_for_workaround_padding_mask * sizeof(int32_t) * 2; + } + + return size; + } + + virtual error_t + collect_tensors_in_workspace_node( + std::unordered_map>>& + workspace_modifications, + int64_t& offset) const override final { + if (attributes.alibi_mask) { + CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(Q, input_names::Q); + int64_t const h_q = Q->second->get_dim()[1]; + auto alibi_slopes_vec = detail::get_alibi_slope(h_q); + workspace_modifications.emplace(alibi_slopes->get_uid(), std::make_tuple(0, offset, alibi_slopes_vec)); + int64_t alibi_slopes_size_padded = ((alibi_slopes_size + 15) / 16 * 16); + offset = offset + alibi_slopes_size_padded; + } + + if (dQ_accum && !dQ_accum->get_is_virtual()) { + if (detail::get_backend_version() < 90600) { + // prior to cuDNN 9.6.0, dQ_accum needed to be memset by frontend + workspace_modifications.emplace(dQ_accum->get_uid(), + std::make_tuple(1, offset, std::vector{(float)dQ_accum_size})); + } else { + workspace_modifications.emplace(dQ_accum->get_uid(), std::make_tuple(2, offset, std::vector())); + } + offset = offset + dQ_accum_size; + } + + if (dK_fullhead && !dK_fullhead->get_is_virtual()) { + workspace_modifications.emplace(dK_fullhead->get_uid(), std::make_tuple(2, offset, std::vector())); + offset = offset + dK_fullhead_size; + } + + if (dV_fullhead && !dV_fullhead->get_is_virtual()) { + workspace_modifications.emplace(dV_fullhead->get_uid(), std::make_tuple(2, offset, std::vector())); + offset = offset + dV_fullhead_size; + } + + if (softmax_sum && !softmax_sum->get_is_virtual()) { + workspace_modifications.emplace(softmax_sum->get_uid(), std::make_tuple(2, offset, std::vector())); + offset = offset + softmax_sum_size; + } + + if (has_workaround_padding_mask) { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Collecting workaround padding mask tensors with batch size " + << batch_size_for_workaround_padding_mask << " with UIDs " + << workaround_padding_mask_seq_len_q->get_uid() << " and " + << workaround_padding_mask_seq_len_kv->get_uid()); + std::vector workaround_padding_mask_seq_len_q_vec(batch_size_for_workaround_padding_mask, + s_q_for_workaround_padding_mask); + std::vector workaround_padding_mask_seq_len_kv_vec(batch_size_for_workaround_padding_mask, + s_kv_for_workaround_padding_mask); + + // reinterpret_cast the int32_t vector data to float vector for workspace_modifications + std::vector workaround_padding_mask_seq_len_q_vec_float( + reinterpret_cast(workaround_padding_mask_seq_len_q_vec.data()), + reinterpret_cast(workaround_padding_mask_seq_len_q_vec.data()) + + batch_size_for_workaround_padding_mask); + std::vector workaround_padding_mask_seq_len_kv_vec_float( + reinterpret_cast(workaround_padding_mask_seq_len_kv_vec.data()), + reinterpret_cast(workaround_padding_mask_seq_len_kv_vec.data()) + + batch_size_for_workaround_padding_mask); + + workspace_modifications.emplace(workaround_padding_mask_seq_len_q->get_uid(), + std::make_tuple(0, offset, workaround_padding_mask_seq_len_q_vec_float)); + offset = offset + batch_size_for_workaround_padding_mask * sizeof(float); + workspace_modifications.emplace(workaround_padding_mask_seq_len_kv->get_uid(), + std::make_tuple(0, offset, workaround_padding_mask_seq_len_kv_vec_float)); + offset = offset + batch_size_for_workaround_padding_mask * sizeof(float); + } + + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"({"tag": "SDPA_BWD"})"_json); + } +#endif +}; + +class UnifiedSDPANode : public SDPANodeBase { + public: + UnifiedSDPANode(SDPA_attributes&& attributes_, detail::Context const& context) + : SDPANodeBase(std::move(attributes_), context) {} + + Type + getType() override final { + return Type::SDPA; + } + + error_t + expand_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for UnifiedSDPANode node " << attributes.name); + + // DO NOT REMOVE + // input data type is needed for: + // - aType of bmm2 + // - dropout scale in pre 8.9.3 + attributes.fill_from_context(this->context); + + //// Optional Attn scale + // In case user provided a scalar value, do a fused scalar. + if (attributes.attn_scale_value.has_value()) { + attributes.inputs[input_names::Attn_scale] = + std::make_shared(attributes.attn_scale_value.value()); + } + + // Optional attention score modifier + if (attributes.attention_score_modifier != nullptr) { + if (!subgraph) init_subgraph(); + subgraph_output = attributes.attention_score_modifier(subgraph, subgraph_output); + } + + // Optional bias + if (attributes.inputs.find(input_names::Bias) != attributes.inputs.end() && + attributes.inputs[input_names::Bias]) { + if (!subgraph) init_subgraph(); + subgraph_output = + attn::score_modifiers::bias(subgraph, subgraph_output, attributes.inputs[input_names::Bias]); + } + + // Optional alibi mask + if (attributes.alibi_mask) { + if (!subgraph) init_subgraph(); + auto h_q = attributes.inputs[input_names::Q]->get_dim()[1]; + subgraph_output = + attn::score_modifiers::alibi_mask(subgraph, subgraph_output, alibi_slopes, h_q, alibi_slopes_size); + } + + // Optional casual masking + if (attributes.left_bound.has_value() || attributes.right_bound.has_value()) { + if (!subgraph) init_subgraph(); + + auto s_q = attributes.inputs[input_names::Q]->get_dim()[2]; + auto s_kv = infer_s_kv(); + auto s_kv_ptr = attributes.inputs.find(input_names::SEQ_LEN_KV) != attributes.inputs.end() + ? attributes.inputs[input_names::SEQ_LEN_KV] + : nullptr; + auto s_q_ptr = attributes.inputs.find(input_names::SEQ_LEN_Q) != attributes.inputs.end() + ? attributes.inputs[input_names::SEQ_LEN_Q] + : nullptr; + auto cu_s_kv_ptr = attributes.inputs.find(input_names::CU_SEQ_LEN_KV) != attributes.inputs.end() + ? attributes.inputs[input_names::CU_SEQ_LEN_KV] + : nullptr; + auto cu_s_q_ptr = attributes.inputs.find(input_names::CU_SEQ_LEN_Q) != attributes.inputs.end() + ? attributes.inputs[input_names::CU_SEQ_LEN_Q] + : nullptr; + + subgraph_output = attn::score_modifiers::sliding_window_mask(subgraph, + subgraph_output, + attributes.diagonal_alignment, + attributes.left_bound, + attributes.right_bound, + s_q, + s_kv, + s_q_ptr, + s_kv_ptr, + cu_s_q_ptr, + cu_s_kv_ptr); + } + + if (subgraph) { + subgraph_output->set_name(attributes.name + "::subgraph_output"); + auto subgraph_node = std::static_pointer_cast(subgraph); + sub_nodes.emplace_back(subgraph_node); + } + + // For BE >= 9.21, express softmax as a UnifiedSoftmaxNode whose backend descriptor + // will be set via CUDNN_ATTR_OPERATION_SDPA_FWD_SOFTMAX_DESC. + // For older versions, stats (if present) is set directly via CUDNN_ATTR_OPERATION_SDPA_FWD_STATSDESC. + auto effective_cudnn_ver = std::min(detail::get_compiled_version(), detail::get_backend_version()); + + auto has_output = [&](auto name) { + auto it = attributes.outputs.find(name); + return it != attributes.outputs.end() && it->second != nullptr; + }; + bool has_softmax_features = has_output(output_names::Stats) || has_output(output_names::Max) || + has_output(output_names::Sum_exp) || attributes.has_sink_token(); + + if (effective_cudnn_ver >= 92100 && has_softmax_features) { + auto b = attributes.inputs[input_names::Q]->get_dim()[0]; + auto h_q = attributes.inputs[input_names::Q]->get_dim()[1]; + auto s_q = attributes.inputs[input_names::Q]->get_dim()[2]; + auto s_kv = infer_s_kv(); + + auto softmax_p = std::make_shared(); + softmax_p->set_is_virtual(true); + softmax_p->set_name(attributes.name + "::softmax_P"); + softmax_p->set_dim({b, h_q, s_q, s_kv}); + softmax_p->set_stride({h_q * s_q * s_kv, s_q * s_kv, s_kv, 1}); + + auto softmax_s = std::make_shared(); + softmax_s->set_is_virtual(true); + softmax_s->set_name(attributes.name + "::softmax_S"); + softmax_s->set_dim({b, h_q, s_q, s_kv}); + softmax_s->set_stride({h_q * s_q * s_kv, s_q * s_kv, s_kv, 1}); + + auto softmax_attrs = Softmax_attributes().set_name(attributes.name + "::softmax"); + softmax_attrs.inputs[Softmax_attributes::input_names::P] = softmax_p; + softmax_attrs.outputs[Softmax_attributes::output_names::S] = softmax_s; + + if (attributes.has_sink_token()) { + softmax_attrs.set_sink(attributes.inputs[input_names::SINK_TOKEN]); + } + if (has_output(output_names::Stats)) { + softmax_attrs.outputs[Softmax_attributes::output_names::Stats] = + attributes.outputs[output_names::Stats]; + } + if (has_output(output_names::Max)) { + softmax_attrs.outputs[Softmax_attributes::output_names::Max] = attributes.outputs[output_names::Max]; + } + if (has_output(output_names::Sum_exp)) { + softmax_attrs.outputs[Softmax_attributes::output_names::Sum_exp] = + attributes.outputs[output_names::Sum_exp]; + } + + softmax_node = std::make_shared(std::move(softmax_attrs), context); + sub_nodes.emplace_back(softmax_node); + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set& uids_involved_in_operations, + std::vector>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map>& tensors) const override final { + CUDNN_FRONTEND_UNUSED(operations); + CUDNN_FE_LOG_LABEL("INFO: Building UnifiedSDPANode operations " << attributes.name << " "); + auto cudnn_ver_error = error_t{error_code_t::GRAPH_NOT_SUPPORTED, "Unified SDPA node requires cuDNN 9.13.1"}; + +#if (CUDNN_VERSION >= 91301) + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(91301, cudnn_ver_error); + auto unified_sdpa_operation = + make_shared_backend_pointer((cudnnBackendDescriptorType_t)CUDNN_BACKEND_OPERATION_SDPA_FWD_DESCRIPTOR); + + auto Q = attributes.inputs.find(SDPA_attributes::input_names::Q)->second; + auto backend_q = tensors[Q->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_QDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_q)); + + auto K = attributes.inputs.find(SDPA_attributes::input_names::K)->second; + auto backend_k = tensors[K->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_KDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_k)); + + auto V = attributes.inputs.find(SDPA_attributes::input_names::V)->second; + auto backend_v = tensors[V->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_VDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_v)); + + auto O = attributes.outputs.find(SDPA_attributes::output_names::O)->second; + auto backend_o = tensors[O->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_ODESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_o)); + + if (softmax_node) { + auto softmax_ver_error = + error_t{error_code_t::GRAPH_NOT_SUPPORTED, "SOFTMAX_DESC in unified SDPA node requires cuDNN 9.21.0"}; +#if (CUDNN_VERSION >= 92100) + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(92100, softmax_ver_error); + + managed_backend_descriptor_t softmax_raw_ops; + std::unordered_set softmax_uids; + std::vector> softmax_ops; + CHECK_CUDNN_FRONTEND_ERROR( + softmax_node->create_cudnn_operations(softmax_uids, softmax_ops, softmax_raw_ops, tensors)); + + auto backend_softmax = softmax_raw_ops[0]->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_SOFTMAX_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_softmax)); + + uids_involved_in_operations.insert(softmax_uids.begin(), softmax_uids.end()); +#else + return softmax_ver_error; +#endif + } else { + auto stats_it = attributes.outputs.find(SDPA_attributes::output_names::Stats); + if (stats_it != attributes.outputs.end() && stats_it->second) { + auto backend_stats = tensors[stats_it->second->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_STATSDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_stats)); + } + } + + auto attn_scale_it = attributes.inputs.find(SDPA_attributes::input_names::Attn_scale); + if (attn_scale_it != attributes.inputs.end()) { + auto backend_scale = tensors[attn_scale_it->second->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_SCALEDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_scale)); + } + + auto block_mask_it = attributes.inputs.find(SDPA_attributes::input_names::Block_mask); + if (block_mask_it != attributes.inputs.end() && block_mask_it->second != nullptr) { + auto block_mask_cudnn_ver_error = + error_t{error_code_t::GRAPH_NOT_SUPPORTED, "Block mask in unified SDPA node requires cuDNN 9.14.0"}; +#if CUDNN_VERSION >= 91400 + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(91400, block_mask_cudnn_ver_error); + auto backend_block_mask = tensors[block_mask_it->second->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_BLOCK_MASK_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_block_mask)); +#else + return block_mask_cudnn_ver_error; +#endif + } + + // Variable-length and paged-attention attributes: + // - paged attention K/V page tables (needs cuDNN >= 9.15.0) + // - per-batch sequence-length tensors (SEQ_LEN_Q / SEQ_LEN_KV; needs cuDNN >= 9.15.0) + // - cumulative sequence-length tensors (CU_SEQ_LEN_Q / CU_SEQ_LEN_KV; needs cuDNN >= 9.24.0). + if (is_paged_k() || is_paged_v() || has_seq_len_q() || has_seq_len_kv() || has_cu_seq_len_q() || + has_cu_seq_len_kv()) { + auto paged_cudnn_ver_error = + error_t{error_code_t::GRAPH_NOT_SUPPORTED, + "Paged attention or per-batch sequence lengths in unified SDPA node require cuDNN 9.15.0"}; +#if (CUDNN_VERSION >= 91500) + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(91500, paged_cudnn_ver_error); + + if (is_paged_k()) { + auto page_table_K = attributes.inputs.find(SDPA_attributes::input_names::Page_table_K)->second; + auto backend_page_table_K = tensors[page_table_K->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_PAGE_TABLE_KDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_page_table_K)); + } + + if (is_paged_v()) { + auto page_table_V = attributes.inputs.find(SDPA_attributes::input_names::Page_table_V)->second; + auto backend_page_table_V = tensors[page_table_V->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_PAGE_TABLE_VDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_page_table_V)); + } + + if (has_seq_len_q()) { + auto seq_len_Q = attributes.inputs.find(SDPA_attributes::input_names::SEQ_LEN_Q)->second; + auto backend_seq_len_Q = tensors[seq_len_Q->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_SEQ_LEN_QDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_seq_len_Q)); + } + + if (has_seq_len_kv()) { + auto seq_len_KV = attributes.inputs.find(SDPA_attributes::input_names::SEQ_LEN_KV)->second; + auto backend_seq_len_KV = tensors[seq_len_KV->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_SEQ_LEN_KVDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_seq_len_KV)); + } + + if (has_cu_seq_len_q() || has_cu_seq_len_kv()) { + auto cu_seq_len_cudnn_ver_error = + error_t{error_code_t::GRAPH_NOT_SUPPORTED, + "Cumulative sequence length tensors in unified SDPA node require cuDNN 9.24.0"}; +#if (CUDNN_VERSION >= 92400) + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(92400, cu_seq_len_cudnn_ver_error); + + if (has_cu_seq_len_q()) { + auto cu_seq_len_Q = attributes.inputs.find(SDPA_attributes::input_names::CU_SEQ_LEN_Q)->second; + auto backend_cu_seq_len_Q = tensors[cu_seq_len_Q->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_CU_SEQ_LEN_QDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_cu_seq_len_Q)); + } + + if (has_cu_seq_len_kv()) { + auto cu_seq_len_KV = attributes.inputs.find(SDPA_attributes::input_names::CU_SEQ_LEN_KV)->second; + auto backend_cu_seq_len_KV = + tensors[cu_seq_len_KV->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_CU_SEQ_LEN_KVDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_cu_seq_len_KV)); + } +#else + return cu_seq_len_cudnn_ver_error; +#endif + } + + // Ignore attributes.max_seq_len_kv, because unified engine doesn't need it (it's harmless if set). + + // Ignore attributes.padding_mask, because unified engine already applies an implicit padding mask + // if either (seq_len_Q and seq_len_KV) or (cu_seq_len_Q and cu_seq_len_KV) are provided. + // We already checked in `SDPA_attributes::validate_sdpa_support_surface()` that padding_mask + // must be true if and only if one of those pairs is set, so we don't need to check it here. +#else + return paged_cudnn_ver_error; +#endif + } + + // Subgraph attributes + if (subgraph) { + auto subgraph_cudnn_ver_error = error_t{error_code_t::GRAPH_NOT_SUPPORTED, + "Subgraph attributes in unified SDPA node requires cuDNN 9.21.0"}; +#if (CUDNN_VERSION >= 92100) + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(92100, subgraph_cudnn_ver_error); + + CHECK_CUDNN_FRONTEND_ERROR(attn::score_modifiers::build_operation_subgraph(subgraph)); + auto subgraph_cudnn = std::static_pointer_cast(subgraph); + auto backend_subgraph = subgraph_cudnn->get_operation_graph()->get_raw_desc(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_SUBGRAPH, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_subgraph)); + + auto subgraph_input_uid = subgraph_input->get_uid(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_SUBGRAPH_INPUT_UID, + CUDNN_TYPE_INT64, + 1, + &subgraph_input_uid)); + + auto subgraph_output_uid = subgraph_output->get_uid(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_SUBGRAPH_OUTPUT_UID, + CUDNN_TYPE_INT64, + 1, + &subgraph_output_uid)); + + // Add non-virtual uids from subgraph to uids_involved_in_operations + uids_involved_in_operations.insert(subgraph_cudnn->variant_pack_uids.begin(), + subgraph_cudnn->variant_pack_uids.end()); +#else + return subgraph_cudnn_ver_error; +#endif + } + + // Set unfuse_fma attribute (for SM100: use __fmul_rn + __fadd_rn instead of ffma2 in softmax) + if (attributes.unfuse_fma) { + auto unfuse_fma_cudnn_ver_error = + error_t{error_code_t::GRAPH_NOT_SUPPORTED, "Unfuse FMA in unified SDPA node requires cuDNN 9.21.0"}; +#if CUDNN_VERSION >= 92100 + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(92100, unfuse_fma_cudnn_ver_error); + bool unfuse_fma_value = true; + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_UNFUSE_FMA, + CUDNN_TYPE_BOOLEAN, + 1, + &unfuse_fma_value)); +#else + return unfuse_fma_cudnn_ver_error; +#endif + } + + // Dropout attributes + if (attributes.dropout_probability.has_value() && attributes.dropout_probability.value() != 0.0f) { + auto dropout_cudnn_ver_error = + error_t{error_code_t::GRAPH_NOT_SUPPORTED, "Dropout in unified SDPA node requires cuDNN 9.21.0"}; +#if (CUDNN_VERSION >= 92100) + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(92100, dropout_cudnn_ver_error); + + float dropout_prob = attributes.dropout_probability.value(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_DROPOUT_PROBABILITY, + CUDNN_TYPE_FLOAT, + 1, + &dropout_prob)); + + auto seed_it = attributes.inputs.find(SDPA_attributes::input_names::Seed); + if (seed_it != attributes.inputs.end() && seed_it->second != nullptr) { + auto backend_seed = tensors[seed_it->second->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_DROPOUT_SEED_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_seed)); + } + + auto offset_it = attributes.inputs.find(SDPA_attributes::input_names::Offset); + if (offset_it != attributes.inputs.end() && offset_it->second != nullptr) { + auto backend_offset = tensors[offset_it->second->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_DROPOUT_OFFSET_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_offset)); + } + + auto rng_dump_it = attributes.outputs.find(SDPA_attributes::output_names::RNG_DUMP); + if (rng_dump_it != attributes.outputs.end() && rng_dump_it->second != nullptr) { + auto backend_rng_dump = tensors[rng_dump_it->second->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SDPA_FWD_DROPOUT_RNG_DUMP_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_rng_dump)); + } +#else + return dropout_cudnn_ver_error; +#endif + } + + // FP8 per-tensor scaling attributes (descale Q/K/V/S, scale S/O inputs; amax S/O outputs). + if (attributes.mma_core_mode == DataType_t::FP8_E4M3 || attributes.mma_core_mode == DataType_t::FP8_E5M2) { + auto fp8_cudnn_ver_error = + error_t{error_code_t::GRAPH_NOT_SUPPORTED, "FP8/MXFP8 for the unified SDPA node requires cuDNN 9.30.0"}; +#if (CUDNN_VERSION >= 93000) + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(93000, fp8_cudnn_ver_error); + + // Wire an optional FP8 scaling/descaling tensor (from either the input or output map) to + // its backend descriptor, no-op if the caller did not provide it. + auto set_fp8_desc = [&](auto const& tensor_map, auto name, cudnnBackendAttributeName_t attr) -> error_t { + auto it = tensor_map.find(name); + if (it != tensor_map.end() && it->second != nullptr) { + auto backend_desc = tensors[it->second->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(unified_sdpa_operation->get_backend_descriptor(), + attr, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_desc)); + } + return {error_code_t::OK, ""}; + }; + + CHECK_CUDNN_FRONTEND_ERROR(set_fp8_desc(attributes.inputs, + SDPA_attributes::input_names::Descale_Q, + CUDNN_ATTR_OPERATION_SDPA_FWD_DESCALE_QDESC)); + CHECK_CUDNN_FRONTEND_ERROR(set_fp8_desc(attributes.inputs, + SDPA_attributes::input_names::Descale_K, + CUDNN_ATTR_OPERATION_SDPA_FWD_DESCALE_KDESC)); + CHECK_CUDNN_FRONTEND_ERROR(set_fp8_desc(attributes.inputs, + SDPA_attributes::input_names::Descale_V, + CUDNN_ATTR_OPERATION_SDPA_FWD_DESCALE_VDESC)); + CHECK_CUDNN_FRONTEND_ERROR(set_fp8_desc(attributes.inputs, + SDPA_attributes::input_names::Descale_S, + CUDNN_ATTR_OPERATION_SDPA_FWD_DESCALE_SDESC)); + CHECK_CUDNN_FRONTEND_ERROR(set_fp8_desc( + attributes.inputs, SDPA_attributes::input_names::Scale_S, CUDNN_ATTR_OPERATION_SDPA_FWD_SCALE_SDESC)); + CHECK_CUDNN_FRONTEND_ERROR(set_fp8_desc( + attributes.inputs, SDPA_attributes::input_names::Scale_O, CUDNN_ATTR_OPERATION_SDPA_FWD_SCALE_ODESC)); + CHECK_CUDNN_FRONTEND_ERROR(set_fp8_desc( + attributes.outputs, SDPA_attributes::output_names::Amax_S, CUDNN_ATTR_OPERATION_SDPA_FWD_AMAX_SDESC)); + CHECK_CUDNN_FRONTEND_ERROR(set_fp8_desc( + attributes.outputs, SDPA_attributes::output_names::Amax_O, CUDNN_ATTR_OPERATION_SDPA_FWD_AMAX_ODESC)); +#else + return fp8_cudnn_ver_error; +#endif // CUDNN_VERSION >= 93000 + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(unified_sdpa_operation->get_backend_descriptor())); + + raw_operations.push_back(unified_sdpa_operation); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; +#else + CUDNN_FRONTEND_UNUSED(uids_involved_in_operations); + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FRONTEND_UNUSED(tensors); + return cudnn_ver_error; +#endif // CUDNN_VERSION >= 91301 + } + + protected: + std::shared_ptr subgraph; // Pre-softmax subgraph + std::shared_ptr subgraph_input; + std::shared_ptr subgraph_output; + std::shared_ptr softmax_node; // Softmax descriptor for BE >= 9.21 + + void + init_subgraph() { + subgraph = std::make_shared(); + auto subgraph_node = std::static_pointer_cast(subgraph); + subgraph_node->context = context; + subgraph_input = std::make_shared(); + subgraph_input->set_is_virtual(true); + subgraph_input->set_name(attributes.name + "::subgraph_input"); + auto b = attributes.inputs[input_names::Q]->get_dim()[0]; + auto h_q = attributes.inputs[input_names::Q]->get_dim()[1]; + auto s_q = attributes.inputs[input_names::Q]->get_dim()[2]; + auto s_kv = infer_s_kv(); + subgraph_input->set_dim({b, h_q, s_q, s_kv}); + subgraph_input->set_stride({h_q * s_q * s_kv, s_q * s_kv, s_kv, 1}); + subgraph_output = subgraph_input; + } +}; + +} // namespace cudnn_frontend::graph diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/sdpa_fp8_bwd.h b/deps/cudnn-frontend/include/cudnn_frontend/node/sdpa_fp8_bwd.h new file mode 100644 index 0000000..0cec5a8 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/sdpa_fp8_bwd.h @@ -0,0 +1,1172 @@ +#pragma once + +#include "../../cudnn_frontend_Heuristics.h" +#include "../../cudnn_frontend_Logging.h" + +#include "../graph_helpers.h" +#include "../node_interface.h" + +#include "diagonal_band_mask.h" +#include "matmul_fp8.h" +#include "pointwise.h" +#include "reduction.h" +#include "softmax.h" +#include "block_scale_dequantize.h" +#include "scaled_dot_product_flash_attention.h" // For promote_1d_index_tensor_to_4d() + +namespace cudnn_frontend::graph { + +class SDPAFP8BackwardNode : public NodeCRTP { + using input_names = SDPA_fp8_backward_attributes::input_names; + using output_names = SDPA_fp8_backward_attributes::output_names; + + private: + mutable bool is_deterministic_algorithm_supported_on_blackwell = false; // Will be edited in pre_validate_node() + + // Promote any 1-D seq_len / ragged-offset index tensors to the 4-D + // [n, 1, 1, 1] form the cuDNN backend requires (see promote_1d_index_tensor_to_4d). + void + promote_index_tensors_to_4d() { + // TODO: Handle CU_SEQ_LEN_Q and CU_SEQ_LEN_KV once fp8 bprop supports these. + for (auto& key : {input_names::SEQ_LEN_Q, input_names::SEQ_LEN_KV}) { + auto const it = attributes.inputs.find(key); + if (it != attributes.inputs.end()) { + promote_1d_index_tensor_to_4d(it->second); + } + } + for (auto& [key, value] : attributes.inputs) { + CUDNN_FRONTEND_UNUSED(key); + if (value != nullptr) { + promote_1d_index_tensor_to_4d(value->get_ragged_offset()); + } + } + for (auto& [key, value] : attributes.outputs) { + CUDNN_FRONTEND_UNUSED(key); + if (value != nullptr) { + promote_1d_index_tensor_to_4d(value->get_ragged_offset()); + } + } + } + + public: + mutable SDPA_fp8_backward_attributes + attributes; // mutable to allow auto-routing to deterministic in pre_validate_node() + + SDPAFP8BackwardNode(SDPA_fp8_backward_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::COMPOSITE; + } + + // Helper function to detect MXFP8 (microscaling FP8) mode + // MXFP8 uses block-wise scale factors with E8M0 data type and F8_128x4 reordering + // When detected, we use block_scale_dequantize before matmuls instead of pointwise descale after + bool + is_mxfp8_scaling() const { + auto descale_q_it = attributes.inputs.find(input_names::Descale_Q); + if (descale_q_it == attributes.inputs.end() || descale_q_it->second == nullptr) { + return false; + } + auto const& descale_q = descale_q_it->second; + return (descale_q->get_data_type() == DataType_t::FP8_E8M0 && + descale_q->get_reordering_type() == TensorReordering_t::F8_128x4); + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating SDPAFP8BackwardNode " << attributes.name); + + RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 90100, + error_code_t::GRAPH_NOT_SUPPORTED, + "sdpa fp8 backward operation is only supported starting cudnn 9.1.0. Please " + "consider upgrading your current version."); + + // Bug workaround for 9.10.0 (TE constraint) + RETURN_CUDNN_FRONTEND_ERROR_IF( + detail::get_backend_version() == 91000, + error_code_t::GRAPH_NOT_SUPPORTED, + "sdpa fp8 backward operation is not supported on cudnn 9.10.0 due to known bugs. " + "Please consider upgrading to 9.10.2 or newer."); + + CHECK_CUDNN_FRONTEND_ERROR(context.populate_sm_version_from_device()); + int32_t const sm_version = context.get_sm_version(); + int32_t const prop_major = sm_version / 10; + RETURN_CUDNN_FRONTEND_ERROR_IF( + prop_major < 9, + error_code_t::GRAPH_NOT_SUPPORTED, + "sdpa fp8 forward operation is only supported on Hopper architecture and newer. Please " + "consider using a newer architecture."); + + // check that Q, K, V, O, stats, dO, dQ, dK, dV tensors has been assigned + // check that dim and strides has been assigned and last stride is 1 +#define CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(port, port_map) \ + { \ + std::shared_ptr tensor_ptr = port_map.at(port); \ + RETURN_CUDNN_FRONTEND_ERROR_IF(tensor_ptr->get_dim().size() != 4, \ + error_code_t::ATTRIBUTE_NOT_SET, \ + "The dim for " + std::string(#port) + " is invalid"); \ + RETURN_CUDNN_FRONTEND_ERROR_IF(tensor_ptr->get_stride().size() != 4, \ + error_code_t::ATTRIBUTE_NOT_SET, \ + "The stride for " + std::string(#port) + " is invalid"); \ + RETURN_CUDNN_FRONTEND_ERROR_IF( \ + tensor_ptr->get_stride()[3] != 1, \ + error_code_t::GRAPH_NOT_SUPPORTED, \ + "The stride for the last dimension corresponding to the embedding size per head should be 1 for " + \ + std::string(#port)); \ + } + + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::Q, attributes.inputs); + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::K, attributes.inputs); + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::V, attributes.inputs); + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::O, attributes.inputs); + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::Stats, attributes.inputs); + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(input_names::dO, attributes.inputs); + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(output_names::dQ, attributes.outputs); + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(output_names::dK, attributes.outputs); + CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE(output_names::dV, attributes.outputs); + +#undef CUDNN_FE_SDPA_VALIDATE_DIM_STRIDE + + // validate backend limitations for the operation + // clang-format off + int64_t s_q = attributes.inputs.at(input_names::Q)->get_dim()[2]; + int64_t s_kv = attributes.inputs.at(input_names::K)->get_dim()[2]; + int64_t h_q = attributes.inputs.at(input_names::Q)->get_dim()[1]; + int64_t h_k = attributes.inputs.at(input_names::K)->get_dim()[1]; + int64_t h_v = attributes.inputs.at(input_names::V)->get_dim()[1]; + int64_t d_qk = attributes.inputs.at(input_names::Q)->get_dim()[3]; + int64_t d_v = attributes.inputs.at(input_names::V)->get_dim()[3]; + + auto const& dq_tensor = attributes.outputs.at(output_names::dQ); + auto const& dq_data_type = dq_tensor->get_data_type(); + auto const& dk_tensor = attributes.outputs.at(output_names::dK); + auto const& dk_data_type = dk_tensor->get_data_type(); + auto const& dv_tensor = attributes.outputs.at(output_names::dV); + auto const& dv_data_type = dv_tensor->get_data_type(); + + auto const& bias_mask = attributes.inputs.find(input_names::Bias); + bool const is_bias = (bias_mask != attributes.inputs.end() && bias_mask->second != nullptr); + + auto const& dropout_mask = attributes.inputs.find(input_names::Dropout_mask); + bool const is_dropout_custom = (dropout_mask != attributes.inputs.end()) && (dropout_mask->second != nullptr); + bool const is_dropout = attributes.dropout_probability.has_value(); + bool const is_ragged = + attributes.inputs.at(input_names::Q)->get_ragged_offset() || + attributes.inputs.at(input_names::K)->get_ragged_offset() || + attributes.inputs.at(input_names::V)->get_ragged_offset() || + attributes.inputs.at(input_names::O)->get_ragged_offset() || + attributes.inputs.at(input_names::Stats)->get_ragged_offset() || + attributes.inputs.at(input_names::dO)->get_ragged_offset() || + attributes.outputs.at(output_names::dQ)->get_ragged_offset() || + attributes.outputs.at(output_names::dK)->get_ragged_offset() || + attributes.outputs.at(output_names::dV)->get_ragged_offset(); + + // validation TODO: + // - validate stats has valid dims + + RETURN_CUDNN_FRONTEND_ERROR_IF((prop_major == 9) && is_ragged, + error_code_t::GRAPH_NOT_SUPPORTED, + "sdpa fp8 backward with THD is not supported on Hopper architecture."); + + // validate basic dimension requirements + if(prop_major >= 10) { + RETURN_CUDNN_FRONTEND_ERROR_IF(((d_qk > 128) || (d_qk % 16 != 0)) && !(d_qk == 192 && d_v == 128), + error_code_t::GRAPH_NOT_SUPPORTED, + "hidden_dim d_qk shoud be less than or equal to 128 and hidden_dim d_qk should be multiple of 16 unless d_qk == 192 and d_v == 128"); + + RETURN_CUDNN_FRONTEND_ERROR_IF(((d_v > 128) || (d_v % 16 != 0)), + error_code_t::GRAPH_NOT_SUPPORTED, + "hidden_dim d_v shoud be less than or equal to 128 and hidden_dim d_v should be multiple of 16"); + } + else { + RETURN_CUDNN_FRONTEND_ERROR_IF((d_qk != 128) || (d_qk % 16 != 0) || (d_v != 128) || (d_v % 16 != 0), + error_code_t::GRAPH_NOT_SUPPORTED, + "hidden_dim shoud be equal to 128 and hidden_dim should be multiple of 16"); + } + RETURN_CUDNN_FRONTEND_ERROR_IF((h_q % h_k != 0) || (h_q % h_v != 0), + error_code_t::GRAPH_NOT_SUPPORTED, + "For group-query attention, number of heads for key and query must be a factor of number of heads for query"); + + // validate options for attn_scale + auto const& attn_scale = attributes.inputs.find(input_names::Attn_scale); + bool const has_attn_scale = (attn_scale != attributes.inputs.end()) && (attn_scale->second != nullptr); + RETURN_CUDNN_FRONTEND_ERROR_IF(has_attn_scale && attributes.attn_scale_value.has_value(), + error_code_t::ATTRIBUTE_NOT_SET, + "attn_scale with tensor and value cannot be set at the same time."); + + // validate options for bias mask + RETURN_CUDNN_FRONTEND_ERROR_IF(is_bias && (bias_mask->second->get_data_type() == DataType_t::BOOLEAN), + error_code_t::GRAPH_NOT_SUPPORTED, + "Bias mask data type cannot be boolean"); + + // validate options for padding mask + auto const& seq_len_q = attributes.inputs.find(input_names::SEQ_LEN_Q); + bool const has_seq_len_q = (seq_len_q != attributes.inputs.end()) && (seq_len_q->second != nullptr); + auto const& seq_len_kv = attributes.inputs.find(input_names::SEQ_LEN_KV); + bool const has_seq_len_kv = (seq_len_kv != attributes.inputs.end()) && (seq_len_kv->second != nullptr); + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.padding_mask && (!has_seq_len_q || !has_seq_len_kv), + error_code_t::ATTRIBUTE_NOT_SET, + "Padding mask requires seq_len_q and seq_len_kv to be set."); + RETURN_CUDNN_FRONTEND_ERROR_IF((!attributes.padding_mask) && (has_seq_len_q || has_seq_len_kv), + error_code_t::ATTRIBUTE_NOT_SET, + "seq_len_q and seq_len_kv needs to be set only if padding mask is enabled."); + + // validate options for dropout mask + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.dropout_probability.has_value() && is_dropout_custom, + error_code_t::ATTRIBUTE_NOT_SET, + "Using both, custom dropout mask and internal-mask generation using dropout probability, is ill-formed."); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.dropout_probability.has_value() && attributes.dropout_probability.value() == 1.0, + error_code_t::ATTRIBUTE_NOT_SET, + "Dropout probability cannot be 1 as corresponding scale wont be well formed."); + + // Validate options for causal_mask_bottom_right + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.has_causal_mask_bottom_right() && detail::get_backend_version() < 90700, + error_code_t::GRAPH_NOT_SUPPORTED, + "For cuDNN version below 9.7.0, bottom right causal masking is not supported."); + + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.has_causal_mask_bottom_right() && prop_major < 10, + error_code_t::GRAPH_NOT_SUPPORTED, + "sdpa fp8 forward operation is only supported on Blackwell architecture and newer. Please " + "consider using a newer architecture."); + + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.has_causal_mask_bottom_right() && s_q > s_kv, + error_code_t::GRAPH_NOT_SUPPORTED, + "Bottom right causal mask does not support s_q > s_kv. Please virtually slice the Q tensor and pass it as s_q == s_kv"); + + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.has_causal_mask_bottom_right() && (is_bias || is_dropout), + error_code_t::GRAPH_NOT_SUPPORTED, + "Bottom right causal mask is only supported with is_bias=False, is_dropout=False."); + + RETURN_CUDNN_FRONTEND_ERROR_IF(attributes.has_causal_mask_bottom_right() && ((s_q % 64 != 0) || (s_kv % 64 != 0)), + error_code_t::GRAPH_NOT_SUPPORTED, + "Bottom right causal mask is only supported with s_q multiple of 64, and s_kv multiple of 64"); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.left_bound.has_value() && (!attributes.padding_mask) && s_q > s_kv, + error_code_t::GRAPH_NOT_SUPPORTED, + "Sliding window attention is only supported with max_s_q <= max_s_kv."); + + // validate that datatype is set for the graph + RETURN_CUDNN_FRONTEND_ERROR_IF(context.get_intermediate_data_type() == DataType_t::NOT_SET, + error_code_t::ATTRIBUTE_NOT_SET, + "Intermediate tensor data type needs to be set as internal tensors require it."); + + // Auto-route to deterministic algorithm for cases where non-deterministic is not supported on Blackwell: + // 1. MXFP8 backward - requires deterministic (STAGES=2) + // 2. FP8 backward with d_qk <= 192 and d_v <= 128 - requires deterministic + bool const is_mxfp8 = is_mxfp8_scaling(); + bool const is_supported_dims = (d_qk <= 192) && (d_v <= 128); + if ((prop_major == 10) && !attributes.is_deterministic_algorithm) { + if (is_mxfp8) { + getLogger() << "[cudnn_frontend] INFO: MXFP8 SDPA backward detected on Blackwell - " + << "auto-routing to deterministic algorithm (non-deterministic not supported for MXFP8)" + << std::endl; + attributes.is_deterministic_algorithm = true; + } else if (is_supported_dims) { + getLogger() << "[cudnn_frontend] INFO: FP8 SDPA backward with d_qk=" << d_qk << ", d_v=" << d_v + << " detected on Blackwell - auto-routing to deterministic algorithm" + << std::endl; + attributes.is_deterministic_algorithm = true; + } + } + + // validate options for deterministic algorithm + if (attributes.is_deterministic_algorithm && (prop_major == 10)) { + RETURN_CUDNN_FRONTEND_ERROR_IF((detail::get_backend_version() < 91900), + error_code_t::GRAPH_NOT_SUPPORTED, + "FP8 deterministic algorithm (required for MXFP8 and d_qk=192/d_v=128) is not supported on Blackwell with cuDNN version below 9.19.0"); + + RETURN_CUDNN_FRONTEND_ERROR_IF(is_dropout, + error_code_t::GRAPH_NOT_SUPPORTED, + "FP8 deterministic algorithm (required for MXFP8 and d_qk=192/d_v=128) is not supported on Blackwell when dropout is enabled"); + + is_deterministic_algorithm_supported_on_blackwell = true; + } + + // if output data type is half or bfloat16 for any of dq, dk, dv, and version is below 9.13 or is not blackwell, return NOT_SUPPORTED + RETURN_CUDNN_FRONTEND_ERROR_IF( + (dq_data_type == DataType_t::HALF || dq_data_type == DataType_t::BFLOAT16 || + dk_data_type == DataType_t::HALF || dk_data_type == DataType_t::BFLOAT16 || + dv_data_type == DataType_t::HALF || dv_data_type == DataType_t::BFLOAT16) && + (detail::get_backend_version() < 91300 || prop_major < 10), + error_code_t::GRAPH_NOT_SUPPORTED, + "sdpa fp8 backward with HALF/BFLOAT16 output is only supported on Blackwell architecture " + "with cuDNN version 9.13.0 and newer."); + + // Validate MXFP8 scale factors if present + if (is_mxfp8_scaling()) { + // MXFP8 requires cuDNN 9.21.0 or later + RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 92100, + error_code_t::GRAPH_NOT_SUPPORTED, + "MXFP8 SDPA requires cuDNN 9.21.0 or later"); + int64_t const b = attributes.inputs.at(input_names::Q)->get_dim()[0]; + + // MXFP8 block size is fixed at 32 + constexpr int64_t block_size = 32; + int64_t const d_qk_scale = (d_qk + block_size - 1) / block_size; + int64_t const d_v_scale = (d_v + block_size - 1) / block_size; + int64_t const s_q_scale = (s_q + block_size - 1) / block_size; + int64_t const s_kv_scale = (s_kv + block_size - 1) / block_size; + + // Validate Descale_Q + auto const& descale_q = attributes.inputs.at(input_names::Descale_Q); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_q->get_data_type() != DataType_t::FP8_E8M0, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_Q to have FP8_E8M0 data type"); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_q->get_reordering_type() != TensorReordering_t::F8_128x4, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_Q to have F8_128x4 reordering"); + + // Validate Descale_Q_T + auto const& descale_q_T = attributes.inputs.at(input_names::Descale_Q_T); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_q_T->get_data_type() != DataType_t::FP8_E8M0, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_Q_T to have FP8_E8M0 data type"); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_q_T->get_reordering_type() != TensorReordering_t::F8_128x4, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_Q_T to have F8_128x4 reordering"); + + // Validate Descale_K + auto const& descale_k = attributes.inputs.at(input_names::Descale_K); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_k->get_data_type() != DataType_t::FP8_E8M0, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_K to have FP8_E8M0 data type"); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_k->get_reordering_type() != TensorReordering_t::F8_128x4, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_K to have F8_128x4 reordering"); + + // Validate Descale_K_T + auto const& descale_k_T = attributes.inputs.at(input_names::Descale_K_T); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_k_T->get_data_type() != DataType_t::FP8_E8M0, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_K_T to have FP8_E8M0 data type"); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_k_T->get_reordering_type() != TensorReordering_t::F8_128x4, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_K_T to have F8_128x4 reordering"); + + // Validate Descale_V + auto const& descale_v = attributes.inputs.at(input_names::Descale_V); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_v->get_data_type() != DataType_t::FP8_E8M0, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_V to have FP8_E8M0 data type"); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_v->get_reordering_type() != TensorReordering_t::F8_128x4, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_V to have F8_128x4 reordering"); + + // Validate Descale_dO + auto const& descale_do = attributes.inputs.at(input_names::Descale_dO); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_do->get_data_type() != DataType_t::FP8_E8M0, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_dO to have FP8_E8M0 data type"); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_do->get_reordering_type() != TensorReordering_t::F8_128x4, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_dO to have F8_128x4 reordering"); + + // Validate Descale_dO_T + auto const& descale_do_T = attributes.inputs.at(input_names::Descale_dO_T); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_do_T->get_data_type() != DataType_t::FP8_E8M0, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_dO_T to have FP8_E8M0 data type"); + RETURN_CUDNN_FRONTEND_ERROR_IF(descale_do_T->get_reordering_type() != TensorReordering_t::F8_128x4, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA requires Descale_dO_T to have F8_128x4 reordering"); + + // Validate dimension consistency for SF_Q: [b, h_q, s_q_padded, d_scale_padded] + auto const& sf_q_dim = descale_q->get_dim(); + RETURN_CUDNN_FRONTEND_ERROR_IF(sf_q_dim[0] != b || sf_q_dim[1] != h_q, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_Q batch/head dimensions must match Q"); + RETURN_CUDNN_FRONTEND_ERROR_IF( + sf_q_dim[3] < d_qk_scale, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_Q d_scale dimension too small (expected >= " + std::to_string(d_qk_scale) + ")"); + + // Validate dimension consistency for SF_Q_T: [b, h_q, s_q_scale_padded, d_padded] + auto const& sf_q_T_dim = descale_q_T->get_dim(); + RETURN_CUDNN_FRONTEND_ERROR_IF(sf_q_T_dim[0] != b || sf_q_T_dim[1] != h_q, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_Q_T batch/head dimensions must match Q"); + RETURN_CUDNN_FRONTEND_ERROR_IF( + sf_q_T_dim[2] < s_q_scale, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_Q_T s_scale dimension too small (expected >= " + std::to_string(s_q_scale) + ")"); + + // Validate dimension consistency for SF_K: [b, h_k, s_kv_padded, d_qk_scale_padded] + auto const& sf_k_dim = descale_k->get_dim(); + RETURN_CUDNN_FRONTEND_ERROR_IF(sf_k_dim[0] != b || sf_k_dim[1] != h_k, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_K batch/head dimensions must match K"); + RETURN_CUDNN_FRONTEND_ERROR_IF( + sf_k_dim[3] < d_qk_scale, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_K d_scale dimension too small (expected >= " + std::to_string(d_qk_scale) + ")"); + + // Validate dimension consistency for SF_K_T: [b, h_k, s_kv_scale_padded, d_qk_padded] + auto const& sf_k_T_dim = descale_k_T->get_dim(); + RETURN_CUDNN_FRONTEND_ERROR_IF(sf_k_T_dim[0] != b || sf_k_T_dim[1] != h_k, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_K_T batch/head dimensions must match K"); + RETURN_CUDNN_FRONTEND_ERROR_IF( + sf_k_T_dim[2] < s_kv_scale, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_K_T s_scale dimension too small (expected >= " + std::to_string(s_kv_scale) + ")"); + + // Validate dimension consistency for SF_V: [b, h_v, s_kv_scale_padded, d_v_padded] + auto const& sf_v_dim = descale_v->get_dim(); + RETURN_CUDNN_FRONTEND_ERROR_IF(sf_v_dim[0] != b || sf_v_dim[1] != h_v, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_V batch/head dimensions must match V"); + RETURN_CUDNN_FRONTEND_ERROR_IF( + sf_v_dim[2] < s_kv_scale, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_V s_scale dimension too small (expected >= " + std::to_string(s_kv_scale) + ")"); + + // Validate dimension consistency for SF_dO: [b, h_q, s_q_padded, d_v_scale_padded] + auto const& sf_do_dim = descale_do->get_dim(); + RETURN_CUDNN_FRONTEND_ERROR_IF(sf_do_dim[0] != b || sf_do_dim[1] != h_q, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_dO batch/head dimensions must match dO"); + RETURN_CUDNN_FRONTEND_ERROR_IF( + sf_do_dim[3] < d_v_scale, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_dO d_scale dimension too small (expected >= " + std::to_string(d_v_scale) + ")"); + + // Validate dimension consistency for SF_dO_T: [b, h_q, s_q_scale_padded, d_scale_padded] + auto const& sf_do_T_dim = descale_do_T->get_dim(); + RETURN_CUDNN_FRONTEND_ERROR_IF(sf_do_T_dim[0] != b || sf_do_T_dim[1] != h_q, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_dO_T batch/head dimensions must match dO"); + RETURN_CUDNN_FRONTEND_ERROR_IF( + sf_do_T_dim[2] < s_q_scale, + error_code_t::ATTRIBUTE_NOT_SET, + "MXFP8 SDPA: Descale_dO_T s_scale dimension too small (expected >= " + std::to_string(s_q_scale) + ")"); + + // Log MXFP8 configuration for debugging + getLogger() << "[cudnn_frontend] INFO: MXFP8 SDPA configuration validated:" << std::endl + << " SF_Q dims: [" << sf_q_dim[0] << ", " << sf_q_dim[1] << ", " << sf_q_dim[2] << ", " + << sf_q_dim[3] << "]" << std::endl + << " SF_Q_T dims: [" << sf_q_T_dim[0] << ", " << sf_q_T_dim[1] << ", " << sf_q_T_dim[2] << ", " + << sf_q_T_dim[3] << "]" << std::endl + << " SF_K dims: [" << sf_k_dim[0] << ", " << sf_k_dim[1] << ", " << sf_k_dim[2] << ", " + << sf_k_dim[3] << "]" << std::endl + << " SF_K_T dims: [" << sf_k_T_dim[0] << ", " << sf_k_T_dim[1] << ", " << sf_k_T_dim[2] << ", " + << sf_k_T_dim[3] << "]" << std::endl + << " SF_V dims: [" << sf_v_dim[0] << ", " << sf_v_dim[1] << ", " << sf_v_dim[2] << ", " + << sf_v_dim[3] << "]" << std::endl + << " SF_dO dims: [" << sf_do_dim[0] << ", " << sf_do_dim[1] << ", " << sf_do_dim[2] << ", " + << sf_do_dim[3] << "]" << std::endl + << " SF_dO_T dims: [" << sf_do_T_dim[0] << ", " << sf_do_T_dim[1] << ", " << sf_do_T_dim[2] << ", " + << sf_do_T_dim[3] << "]" << std::endl + << " Expected d_qk_scale: " << d_qk_scale << ", d_v_scale: " << d_v_scale + << ", s_q_scale: " << s_q_scale << ", s_kv_scale: " << s_kv_scale << std::endl; + } + + // validate options for sink token + auto const& sink_token = attributes.inputs.find(input_names::SINK_TOKEN); + bool const has_sink_token = (sink_token != attributes.inputs.end()) && (sink_token->second != nullptr); + auto const& dsink_token = attributes.outputs.find(output_names::DSINK_TOKEN); + bool const has_dsink_token = (dsink_token != attributes.outputs.end()) && (dsink_token->second != nullptr); + RETURN_CUDNN_FRONTEND_ERROR_IF(has_dsink_token && !has_sink_token, + error_code_t::ATTRIBUTE_NOT_SET, + "If dSink_token output is requested, sink_token input must also be set."); + + return {error_code_t::OK, ""}; + } + + error_t + infer_properties_node() override final { + // If a 1-D length/offset index tensor [n] is supplied, promote it to the 4-D + // [n, 1, 1, 1] form the cuDNN backend requires. + promote_index_tensors_to_4d(); + + return {error_code_t::OK, ""}; + } + + error_t + expand_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for Scaled_dot_product_flash_attention node " + << attributes.name); + + attributes.fill_from_context(context); + + // Gather dim to fill properties of virtual tensors + auto const& q_dim = attributes.inputs[input_names::Q]->get_dim(); + auto b = q_dim[0]; + auto h_q = q_dim[1]; + auto s_q = q_dim[2]; + auto d_qk = q_dim[3]; + auto const& k_dim = attributes.inputs[input_names::K]->get_dim(); + auto h_k = k_dim[1]; + auto s_kv = k_dim[2]; + auto const& v_dim = attributes.inputs[input_names::V]->get_dim(); + auto h_v = v_dim[1]; + auto d_v = v_dim[3]; + + bool const use_mxfp8 = is_mxfp8_scaling(); + std::shared_ptr eff_Q, eff_K, eff_V, eff_dO; + + // cuDNN frontend API attention requires Q, K, V where + // Q = {b, h_q, s_q, d_qk} + // K = {b, h_k, s_kv, d_qk} + // V = {b, h_v, s_kv, d_v} + // but cuDNN backend API attention requires Q, KT, VT + // Q = {b, h_q, s_q, d_qk} + // KT = {b, h_k, d_qk, s_kv} + // VT = {b, h_v, d_v, s_kv} + // So the code below maps the K->KT and V->VT + if (use_mxfp8) { + // MXFP8 2-kernel bprop: Each tensor is dequantized separately for each matmul it participates in. + // Q, K, dO each need two dequantizations with different scale factors. + // O and dO are used directly (as f16) for the dO*O computation without dequantization. + + // K for BMM1 (Q@K.T): Transpose K and SF_K, then dequantize + std::vector kt_dim = attributes.inputs[input_names::K]->get_dim(); + std::vector kt_stride = attributes.inputs[input_names::K]->get_stride(); + std::swap(kt_dim[2], kt_dim[3]); + std::swap(kt_stride[2], kt_stride[3]); + attributes.inputs[input_names::K]->set_dim(kt_dim); + attributes.inputs[input_names::K]->set_stride(kt_stride); + + auto& sf_k = attributes.inputs[input_names::Descale_K]; + std::vector sf_k_dim = sf_k->get_dim(); + std::vector sf_k_stride = sf_k->get_stride(); + std::swap(sf_k_dim[2], sf_k_dim[3]); + std::swap(sf_k_stride[2], sf_k_stride[3]); + sf_k->set_dim(sf_k_dim); + sf_k->set_stride(sf_k_stride); + + auto dequant_k_dqk_attrs = Block_scale_dequantize_attributes() + .set_name("DQ_K_dqk") + .set_block_size({1, 32}); + auto k_dqk_dequant = std::make_shared(); + k_dqk_dequant->set_is_virtual(true); + k_dqk_dequant->set_dim(kt_dim); + k_dqk_dequant->set_stride(kt_stride); + block_scale_dequantize(attributes.inputs[input_names::K], sf_k, dequant_k_dqk_attrs, k_dqk_dequant); + eff_K = k_dqk_dequant; + + // V for dO@V.T: Transpose V and SF_V, then dequantize + std::vector vt_dim = attributes.inputs[input_names::V]->get_dim(); + std::vector vt_stride = attributes.inputs[input_names::V]->get_stride(); + std::swap(vt_dim[2], vt_dim[3]); + std::swap(vt_stride[2], vt_stride[3]); + attributes.inputs[input_names::V]->set_dim(vt_dim); + attributes.inputs[input_names::V]->set_stride(vt_stride); + + auto& sf_v = attributes.inputs[input_names::Descale_V]; + std::vector sf_v_dim = sf_v->get_dim(); + std::vector sf_v_stride = sf_v->get_stride(); + std::swap(sf_v_dim[2], sf_v_dim[3]); + std::swap(sf_v_stride[2], sf_v_stride[3]); + sf_v->set_dim(sf_v_dim); + sf_v->set_stride(sf_v_stride); + + auto dequant_v_dv_attrs = Block_scale_dequantize_attributes() + .set_name("DQ_V_dv") + .set_block_size({1, 32}); + auto v_dv_dequant = std::make_shared(); + v_dv_dequant->set_is_virtual(true); + v_dv_dequant->set_dim(vt_dim); + v_dv_dequant->set_stride(vt_stride); + block_scale_dequantize(attributes.inputs[input_names::V], sf_v, dequant_v_dv_attrs, v_dv_dequant); + eff_V = v_dv_dequant; + + // Q for BMM1 (Q@K.T): Dequantize Q with Descale_Q + auto dequant_q_dqk_attrs = Block_scale_dequantize_attributes() + .set_name("DQ_Q_dqk") + .set_block_size({1, 32}); + auto q_dqk_dequant = std::make_shared(); + q_dqk_dequant->set_is_virtual(true); + q_dqk_dequant->set_dim(attributes.inputs[input_names::Q]->get_dim()); + q_dqk_dequant->set_stride(attributes.inputs[input_names::Q]->get_stride()); + block_scale_dequantize(attributes.inputs[input_names::Q], + attributes.inputs[input_names::Descale_Q], + dequant_q_dqk_attrs, + q_dqk_dequant); + eff_Q = q_dqk_dequant; + + // dO for dO@V.T: Dequantize dO with Descale_dO + auto dequant_dO_dv_attrs = Block_scale_dequantize_attributes() + .set_name("DQ_dO_dv") + .set_block_size({1, 32}); + auto dO_dv_dequant = std::make_shared(); + dO_dv_dequant->set_is_virtual(true); + dO_dv_dequant->set_dim(attributes.inputs[input_names::dO]->get_dim()); + dO_dv_dequant->set_stride(attributes.inputs[input_names::dO]->get_stride()); + block_scale_dequantize(attributes.inputs[input_names::dO], + attributes.inputs[input_names::Descale_dO], + dequant_dO_dv_attrs, + dO_dv_dequant); + eff_dO = dO_dv_dequant; + + // Note: O and dO for the dO*O computation are used as raw f16 inputs (no MXFP8 dequant). + // The separate dequantizations for dO_seq, K_seq, Q_seq are done inline at their usage sites. + } else { + // Non-MXFP8 + std::vector temp_vec; + + temp_vec = attributes.inputs[input_names::K]->get_dim(); + std::swap(temp_vec[2], temp_vec[3]); + attributes.inputs[input_names::K]->set_dim(temp_vec); + + temp_vec = attributes.inputs[input_names::K]->get_stride(); + std::swap(temp_vec[2], temp_vec[3]); + attributes.inputs[input_names::K]->set_stride(temp_vec); + + temp_vec = attributes.inputs[input_names::V]->get_dim(); + std::swap(temp_vec[2], temp_vec[3]); + attributes.inputs[input_names::V]->set_dim(temp_vec); + + temp_vec = attributes.inputs[input_names::V]->get_stride(); + std::swap(temp_vec[2], temp_vec[3]); + attributes.inputs[input_names::V]->set_stride(temp_vec); + + eff_Q = attributes.inputs[input_names::Q]; + eff_K = attributes.inputs[input_names::K]; + eff_V = attributes.inputs[input_names::V]; + eff_dO = attributes.inputs[input_names::dO]; + } + + std::shared_ptr rng_output; + + auto mul_attributes = Pointwise_attributes().set_mode(PointwiseMode_t::MUL); + + // if dropout_prob is used, then the node passes scale and scale inverse + // if dropout_mask is used, then the user passes scale and scale_inverse + bool is_dropout_prob = (attributes.dropout_probability.has_value()); + bool is_dropout_mask = (attributes.inputs[input_names::Dropout_mask] != nullptr); + if (is_dropout_prob) { + float dropout_scale_value = 1.0f / (1.0f - attributes.dropout_probability.value()); + float dropout_scale_inv_value = (1.0f - attributes.dropout_probability.value()); + + attributes.inputs[input_names::Dropout_scale] = std::make_shared(dropout_scale_value); + attributes.inputs[input_names::Dropout_scale_inv] = + std::make_shared(dropout_scale_inv_value); + } + + // --------------RNG node-------------------- + + if (is_dropout_prob) { + rng_output = rng(attributes.inputs[input_names::Seed], + attributes.inputs[input_names::Offset], + Rng_attributes() + .set_name("rng") + .set_distribution(RngDistribution_t::BERNOULLI) + .set_bernoulli_probability(1.0f - attributes.dropout_probability.value())); + rng_output->set_dim({b, h_q, s_q, s_kv}).set_stride({h_q * s_q * s_kv, s_q * s_kv, s_kv, 1}); + } else if (is_dropout_mask) { + rng_output = attributes.inputs[input_names::Dropout_mask]; + } + + //// dO * O + // For MXFP8: dO_f16 and O (f16) are used directly (no block dequantization needed) + // For non-MXFP8 (17-arg API): dO and O (FP8) are used (per-tensor descale applied after reduction) + mul_attributes.set_name("mul_dO_O"); + auto last_output = use_mxfp8 + ? pointwise(attributes.inputs[input_names::dO_f16], attributes.inputs[input_names::O], mul_attributes) + : pointwise(attributes.inputs[input_names::dO], attributes.inputs[input_names::O], mul_attributes); + + // reduce(dO) + last_output = + reduction(last_output, Reduction_attributes().set_name("reduce_dO").set_mode(ReductionMode_t::ADD)); + last_output->set_dim({b, h_q, s_q, 1}).set_stride({h_q * s_q, s_q, 1, 1}); + + // Descale dO (only for non-MXFP8 per-tensor scaling) + if (!use_mxfp8) { + mul_attributes.set_name("descale_dO"); + last_output = pointwise(last_output, attributes.inputs.at(input_names::Descale_dO), mul_attributes); + last_output->set_dim({b, h_q, s_q, 1}).set_stride({h_q * s_q, s_q, 1, 1}); + } + + // Descale O (only for non-MXFP8 per-tensor scaling) + if (!use_mxfp8) { + mul_attributes.set_name("descale_O"); + last_output = pointwise(last_output, attributes.inputs.at(input_names::Descale_O), mul_attributes); + } + + // dSink_token computation: dSink = sum(exp(sink - stats) * D) over batch and sequence + // Note: The backend kernel applies the negative sign internally. + auto const& sink_token_it = attributes.inputs.find(input_names::SINK_TOKEN); + auto const& dsink_token_it = attributes.outputs.find(output_names::DSINK_TOKEN); + if (dsink_token_it != attributes.outputs.end() && dsink_token_it->second != nullptr && + sink_token_it != attributes.inputs.end() && sink_token_it->second != nullptr) { + // sub_sink = sink - stats + auto sub_sink = pointwise(sink_token_it->second, + attributes.inputs[input_names::Stats], + Pointwise_attributes().set_name("sub_sink_stats").set_mode(PointwiseMode_t::SUB)); + sub_sink->set_dim({b, h_q, s_q, 1}).set_stride({h_q * s_q, s_q, 1, 1}); + + // exp_sink = exp(sub_sink) + auto exp_sink = + pointwise(sub_sink, Pointwise_attributes().set_name("exp_sink").set_mode(PointwiseMode_t::EXP)); + exp_sink->set_dim({b, h_q, s_q, 1}).set_stride({h_q * s_q, s_q, 1, 1}); + + // per_token_grad = exp_sink * last_output (D) + auto per_token_grad = pointwise( + exp_sink, last_output, Pointwise_attributes().set_name("mul_exp_sink_D").set_mode(PointwiseMode_t::MUL)); + per_token_grad->set_dim({b, h_q, s_q, 1}).set_stride({h_q * s_q, s_q, 1, 1}); + + // dSink = reduce(per_token_grad) over batch and sequence dimensions + reduction(per_token_grad, + Reduction_attributes().set_name("reduce_dSink").set_mode(ReductionMode_t::ADD), + dsink_token_it->second); + } + + // softmax_sum = last_output * dropout_scale + // Note: When sink_token is enabled and dropout is NOT present, we still need a MUL operation + // in the softmax path so the backend pattern matcher (fortAttentionBackwardRuntimeFusionEngine) + // can correctly distinguish between the dSink chain (MUL→REDUCE) and the softmax path (MUL→SUB). + // Without this, the fallback at lines 3786-3796 incorrectly identifies dSink MUL as the softmax path. + bool const has_sink_token = (sink_token_it != attributes.inputs.end() && sink_token_it->second != nullptr); + if (attributes.inputs[input_names::Dropout_scale_inv]) { + last_output = pointwise(last_output, + attributes.inputs[input_names::Dropout_scale_inv], + Pointwise_attributes().set_name("scale_dropout_inv").set_mode(PointwiseMode_t::MUL)); + } else if (has_sink_token) { + // Add identity MUL (multiply by 1) to create MUL→SUB pattern for pattern matcher + auto one_tensor = std::make_shared(1.0f); + last_output = pointwise(last_output, + one_tensor, + Pointwise_attributes().set_name("identity_for_sink").set_mode(PointwiseMode_t::MUL)); + last_output->set_dim({b, h_q, s_q, 1}).set_stride({h_q * s_q, s_q, 1, 1}); + } + auto softmax_sum = last_output; + + //// Q * K + auto bmm_Q_K_attributes = Matmul_attributes().set_name("bmm_Q_K") + .set_m_override(attributes.inputs[input_names::SEQ_LEN_Q]) + .set_n_override(attributes.inputs[input_names::SEQ_LEN_KV]); + auto last_dV = matmul(eff_Q, eff_K, bmm_Q_K_attributes); + + //// Optional Attn scale + // In case user provided a scalar value, do a fused scalar. + if (attributes.attn_scale_value.has_value()) { + attributes.inputs[input_names::Attn_scale] = + std::make_shared(attributes.attn_scale_value.value()); + } + + // If attn scale present, add a pointwise mul node + if (auto attn_scale_it = attributes.inputs.find(input_names::Attn_scale); attn_scale_it != attributes.inputs.end()) { + mul_attributes.set_name("attn_scale"); + last_dV = pointwise(last_dV, attn_scale_it->second, mul_attributes); + } + + // Descale Q (only for non-MXFP8 per-tensor scaling) + if (!use_mxfp8) { + mul_attributes.set_name("descale_q"); + last_dV = pointwise(last_dV, attributes.inputs.at(input_names::Descale_Q), mul_attributes); + } + + // Descale K (only for non-MXFP8 per-tensor scaling) + if (!use_mxfp8) { + mul_attributes.set_name("descale_k"); + last_dV = pointwise(last_dV, attributes.inputs.at(input_names::Descale_K), mul_attributes); + } + + // (optional) last_dV = last_dV + bias + if (auto bias_it = attributes.inputs.find(input_names::Bias); bias_it != attributes.inputs.end()) { + last_dV = pointwise(last_dV, + bias_it->second, + Pointwise_attributes().set_name("add_bias").set_mode(PointwiseMode_t::ADD)); + } + + // (optional) Apply padding mask + if (attributes.padding_mask) { + auto row_idx_output = pointwise(last_dV, + Pointwise_attributes() + .set_name("gen_row_idx_padding") + .set_mode(PointwiseMode_t::GEN_INDEX) + .set_axis(2) + .set_compute_data_type(DataType_t::INT32)); + row_idx_output->set_data_type(DataType_t::INT32); + + auto col_idx_output = pointwise(last_dV, + Pointwise_attributes() + .set_name("gen_col_idx_padding") + .set_mode(PointwiseMode_t::GEN_INDEX) + .set_axis(3) + .set_compute_data_type(DataType_t::INT32)); + col_idx_output->set_data_type(DataType_t::INT32); + + auto row_mask_output = pointwise(row_idx_output, + attributes.inputs[input_names::SEQ_LEN_Q], + Pointwise_attributes() + .set_name("lt_row_sq_padding") + .set_mode(PointwiseMode_t::CMP_LT) + .set_compute_data_type(DataType_t::BOOLEAN)); + row_mask_output->set_data_type(DataType_t::BOOLEAN); + + auto col_mask_output = pointwise(col_idx_output, + attributes.inputs[input_names::SEQ_LEN_KV], + Pointwise_attributes() + .set_name("lt_col_skv_padding") + .set_mode(PointwiseMode_t::CMP_LT) + .set_compute_data_type(DataType_t::BOOLEAN)); + col_mask_output->set_data_type(DataType_t::BOOLEAN); + + auto padding_mask_output = pointwise(row_mask_output, + col_mask_output, + Pointwise_attributes() + .set_name("and_row_col_padding") + .set_mode(PointwiseMode_t::LOGICAL_AND) + .set_compute_data_type(DataType_t::BOOLEAN)); + padding_mask_output->set_data_type(DataType_t::BOOLEAN); + + // Use a smaller value of neg infinity so that the softmax stats for rows that are fully padded dont + // go towards NaNs/Infs when multipled by the numerous scale/descale + auto negative_inf_padding = std::make_shared(attn::score_modifiers::get_negative_inf_value()); + + last_dV = + pointwise(last_dV, + negative_inf_padding, + padding_mask_output, + Pointwise_attributes().set_name("select_padding").set_mode(PointwiseMode_t::BINARY_SELECT)); + } + + // Apply (bottom-right) causal masking (with right bound) and/or set the left bound + if (attributes.left_bound.has_value() || attributes.right_bound.has_value()) { + auto graph_ = std::make_shared(); + std::shared_ptr node_ = std::static_pointer_cast(graph_); + node_->context = this->context; + + auto s_kv_ptr = attributes.inputs.find(input_names::SEQ_LEN_KV) != attributes.inputs.end() + ? attributes.inputs[input_names::SEQ_LEN_KV] + : nullptr; + auto s_q_ptr = attributes.inputs.find(input_names::SEQ_LEN_Q) != attributes.inputs.end() + ? attributes.inputs[input_names::SEQ_LEN_Q] + : nullptr; + + last_dV = attn::score_modifiers::sliding_window_mask(graph_, + last_dV, + attributes.diagonal_alignment, + attributes.left_bound, + attributes.right_bound, + s_q, + s_kv, + s_q_ptr, + s_kv_ptr, + /*cu_s_q_ptr=*/nullptr, + /*cu_s_kv_ptr=*/nullptr); + sub_nodes.emplace_back(std::move(node_)); + } + + //// Apply Softmax + // last_dV = last_dV - stats + last_dV = pointwise(last_dV, + attributes.inputs[input_names::Stats], + Pointwise_attributes().set_name("sub_dV_Stats").set_mode(PointwiseMode_t::SUB)); + + // last_dV = exp(last_dV) + last_dV = pointwise(last_dV, Pointwise_attributes().set_name("exp_dV").set_mode(PointwiseMode_t::EXP)); + auto exp_S = last_dV; + + // (optional) last_dV = last_dV * dropout rng_output + if (is_dropout_prob || is_dropout_mask) { + last_dV = + pointwise(last_dV, + rng_output, + Pointwise_attributes().set_name("mul_p_dropout_mask").set_mode(PointwiseMode_t::MUL)); + } + + // (optional) last_dV = last_dV * dropout_scale + if (attributes.inputs[input_names::Dropout_scale]) { + last_dV = + pointwise(last_dV, + attributes.inputs[input_names::Dropout_scale], + Pointwise_attributes().set_name("mul_dS_dropout_scale").set_mode(PointwiseMode_t::MUL)); + } + + // Scale S (only for non-MXFP8 per-tensor scaling; MXFP8 keeps P in fp32) + if (!use_mxfp8) { + mul_attributes.set_name("scale_S"); + last_dV = pointwise(last_dV, attributes.inputs.at(input_names::Scale_S), mul_attributes); + last_dV->set_data_type(attributes.inputs.at(input_names::Q)->get_data_type()); + } + + // Reshape P + last_dV = reshape(last_dV, Reshape_attributes().set_name("S_transpose")); + last_dV->set_name("S_T").set_dim({b, h_q, s_kv, s_q}).set_stride({h_q * s_q * s_kv, s_q * s_kv, 1, s_kv}); + if (!use_mxfp8) { + last_dV->set_data_type(attributes.inputs[input_names::Q]->get_data_type()); + } + + //// P.T @ dO -> dV + if (use_mxfp8) { + // MXFP8: Dequantize dO_T with Descale_dO_T for P.T@dO + auto dequant_dO_seq_attrs = Block_scale_dequantize_attributes() + .set_name("DQ_dO_seq") + .set_block_size({1, 32}); + auto dO_seq_dequant = std::make_shared(); + dO_seq_dequant->set_is_virtual(true); + dO_seq_dequant->set_dim(attributes.inputs[input_names::dO_T]->get_dim()); + dO_seq_dequant->set_stride(attributes.inputs[input_names::dO_T]->get_stride()); + block_scale_dequantize(attributes.inputs[input_names::dO_T], + attributes.inputs[input_names::Descale_dO_T], + dequant_dO_seq_attrs, + dO_seq_dequant); + + auto bmm_S_T_dO_attributes = Matmul_attributes().set_name("bmm_S_T_dO") + .set_m_override(attributes.inputs[input_names::SEQ_LEN_KV]) + .set_k_override(attributes.inputs[input_names::SEQ_LEN_Q]); + if (h_q == h_v) { + matmul(last_dV, dO_seq_dequant, bmm_S_T_dO_attributes, attributes.outputs[output_names::dV]); + } else { + auto dV_fullhead = matmul(last_dV, dO_seq_dequant, bmm_S_T_dO_attributes); + dV_fullhead->set_dim({b, h_q, s_kv, d_v}); + dV_fullhead->set_stride({h_q * s_kv * d_v, s_kv * d_v, d_v, 1}); + dV_fullhead->set_data_type(DataType_t::FLOAT); + reduction(dV_fullhead, + Reduction_attributes().set_name("red_dV_head").set_mode(ReductionMode_t::ADD), + attributes.outputs[output_names::dV]); + } + + auto const& amax_dv = attributes.outputs.at(output_names::Amax_dV); + if (amax_dv != nullptr) { + auto amax_dv_attributes = + Reduction_attributes().set_name("amax_dV").set_mode(ReductionMode_t::AMAX); + reduction(attributes.outputs[output_names::dV], amax_dv_attributes, amax_dv); + } + } else { + matmul_fp8(last_dV, + attributes.inputs[input_names::dO], + attributes.inputs[input_names::Descale_S], + attributes.inputs[input_names::Descale_dO], + attributes.inputs[input_names::Scale_dV], + Matmul_fp8_attributes().set_name("bmm_S_T_dO") + .set_m_override(attributes.inputs[input_names::SEQ_LEN_KV]) + .set_k_override(attributes.inputs[input_names::SEQ_LEN_Q]), + attributes.outputs[output_names::dV], + attributes.outputs[output_names::Amax_dV]); + } + + //// dO * V_T + auto bmm_dO_V_T_attributes = Matmul_attributes().set_name("bmm_dO_V_T") + .set_m_override(attributes.inputs[input_names::SEQ_LEN_Q]) + .set_n_override(attributes.inputs[input_names::SEQ_LEN_KV]); + last_output = matmul(eff_dO, eff_V, bmm_dO_V_T_attributes); + + // Descale dO (only for non-MXFP8 per-tensor scaling) + if (!use_mxfp8) { + mul_attributes.set_name("descale_dO"); + last_output = pointwise(last_output, attributes.inputs.at(input_names::Descale_dO), mul_attributes); + } + + // Descale V (only for non-MXFP8 per-tensor scaling) + if (!use_mxfp8) { + mul_attributes.set_name("descale_V"); + last_output = pointwise(last_output, attributes.inputs.at(input_names::Descale_V), mul_attributes); + } + + // dP = last_output - softmax_sum + auto dP = pointwise(last_output, + softmax_sum, + Pointwise_attributes().set_name("sub_dP_softmax_sum").set_mode(PointwiseMode_t::SUB)); + + // dP = dP * exp_S + mul_attributes.set_name("mul_dP_exp_S"); + dP = pointwise(dP, exp_S, mul_attributes); + + // (optional) dP = dP * dropout_scale + if (attributes.inputs[input_names::Dropout_scale]) { + dP = + pointwise(dP, + attributes.inputs[input_names::Dropout_scale], + Pointwise_attributes().set_name("mul_dS_dropout_scale").set_mode(PointwiseMode_t::MUL)); + } + + // if (attributes.outputs[output_names::dBias]) { + // reduction(dP, + // Reduction_attributes().set_name("red_dP_dBias").set_mode(ReductionMode_t::ADD), + // attributes.outputs[output_names::dBias]); + // } + + // (optional) dP = dP * attn_scale + if (auto attn_scale_it = attributes.inputs.find(input_names::Attn_scale); attn_scale_it != attributes.inputs.end()) { + mul_attributes.set_name("mul_dS_attn_scale"); + dP = pointwise(dP, attn_scale_it->second, mul_attributes); + } + + // Amax dP and Scale dP (only for non-MXFP8 per-tensor scaling; MXFP8 keeps dS in fp32) + if (!use_mxfp8) { + auto amax_attributes = Reduction_attributes().set_name("amax_dP").set_mode(ReductionMode_t::AMAX); + reduction(dP, amax_attributes, attributes.outputs.at(output_names::Amax_dP)); + + mul_attributes.set_name("scale_dP"); + dP = pointwise(dP, attributes.inputs.at(input_names::Scale_dP), mul_attributes); + dP->set_data_type(attributes.inputs.at(input_names::dO)->get_data_type()); + } + + //// dS @ K -> dQ + if (use_mxfp8) { + // MXFP8: Dequantize K_T with Descale_K_T (seq-dimension scaling) for dS@K -> dQ + auto dequant_K_seq_attrs = Block_scale_dequantize_attributes() + .set_name("DQ_K_seq") + .set_block_size({1, 32}); + auto K_seq_dequant = std::make_shared(); + K_seq_dequant->set_is_virtual(true); + K_seq_dequant->set_dim(attributes.inputs[input_names::K_T]->get_dim()); + K_seq_dequant->set_stride(attributes.inputs[input_names::K_T]->get_stride()); + + int64_t s_kv_scale_padded = ((s_kv + 127) / 128) * 4; + auto& sf_k_T = attributes.inputs[input_names::Descale_K_T]; + sf_k_T->set_stride({h_k * s_kv_scale_padded * 128, s_kv_scale_padded * 128, b * h_k * s_kv_scale_padded * 128, 1}); + + block_scale_dequantize(attributes.inputs[input_names::K_T], + attributes.inputs[input_names::Descale_K_T], + dequant_K_seq_attrs, + K_seq_dequant); + + auto bmm_dS_K_attributes = Matmul_attributes().set_name("bmm_dS_K") + .set_m_override(attributes.inputs[input_names::SEQ_LEN_Q]) + .set_k_override(attributes.inputs[input_names::SEQ_LEN_KV]); + matmul(dP, K_seq_dequant, bmm_dS_K_attributes, attributes.outputs[output_names::dQ]); + + auto const& amax_dq = attributes.outputs.at(output_names::Amax_dQ); + if (amax_dq != nullptr) { + auto amax_dq_attributes = + Reduction_attributes().set_name("amax_dQ").set_mode(ReductionMode_t::AMAX); + reduction(attributes.outputs[output_names::dQ], amax_dq_attributes, amax_dq); + } + } else { + auto const& kt_dim = attributes.inputs[input_names::K]->get_dim(); + auto const& kt_stride = attributes.inputs[input_names::K]->get_stride(); + + auto K = reshape(attributes.inputs[input_names::K], Reshape_attributes().set_name("reshape_K")); + K->set_dim({kt_dim[0], kt_dim[1], kt_dim[3], kt_dim[2]}) + .set_stride({kt_stride[0], kt_stride[1], kt_stride[3], kt_stride[2]}); + + auto bmm_dP_K_attributes = Matmul_fp8_attributes().set_name("bmm_dP_K") + .set_m_override(attributes.inputs[input_names::SEQ_LEN_Q]) + .set_k_override(attributes.inputs[input_names::SEQ_LEN_KV]); + matmul_fp8(dP, + K, + attributes.inputs[input_names::Descale_dP], + attributes.inputs[input_names::Descale_K], + attributes.inputs[input_names::Scale_dQ], + bmm_dP_K_attributes, + attributes.outputs[output_names::dQ], + attributes.outputs[output_names::Amax_dQ]); + } + + //// dS.T * Q (transpose dS -> dS_reshape) + auto dP_T_attributes = Reshape_attributes().set_name("dP_T"); + auto dP_T = reshape(dP, dP_T_attributes); + if (!use_mxfp8) { + dP_T->set_data_type(attributes.inputs.at(input_names::dO)->get_data_type()); + } + dP_T->set_name("dP_T").set_dim({b, h_q, s_kv, s_q}).set_stride({h_q * s_q * s_kv, s_q * s_kv, 1, s_kv}); + + if (use_mxfp8) { + // MXFP8: Dequantize Q_T with Descale_Q_T (seq-dimension scaling) for dS.T@Q -> dK + auto dequant_Q_seq_attrs = Block_scale_dequantize_attributes() + .set_name("DQ_Q_seq") + .set_block_size({1, 32}); + auto Q_seq_dequant = std::make_shared(); + Q_seq_dequant->set_is_virtual(true); + Q_seq_dequant->set_dim(attributes.inputs[input_names::Q_T]->get_dim()); + Q_seq_dequant->set_stride(attributes.inputs[input_names::Q_T]->get_stride()); + + int64_t s_q_scale_padded = ((s_q + 127) / 128) * 4; + auto& sf_q_T = attributes.inputs[input_names::Descale_Q_T]; + sf_q_T->set_stride({h_q * s_q_scale_padded * 128, s_q_scale_padded * 128, b * h_q * s_q_scale_padded * 128, 1}); + + block_scale_dequantize(attributes.inputs[input_names::Q_T], + attributes.inputs[input_names::Descale_Q_T], + dequant_Q_seq_attrs, + Q_seq_dequant); + + auto bmm_dS_T_Q_attributes = Matmul_attributes().set_name("bmm_dS_T_Q") + .set_m_override(attributes.inputs[input_names::SEQ_LEN_KV]) + .set_k_override(attributes.inputs[input_names::SEQ_LEN_Q]); + if (h_q == h_k) { + matmul(dP_T, Q_seq_dequant, bmm_dS_T_Q_attributes, attributes.outputs[output_names::dK]); + } else { + auto dK_fullhead = matmul(dP_T, Q_seq_dequant, bmm_dS_T_Q_attributes); + dK_fullhead->set_dim({b, h_q, s_kv, d_qk}); + dK_fullhead->set_stride({h_q * s_kv * d_qk, s_kv * d_qk, d_qk, 1}); + dK_fullhead->set_data_type(DataType_t::FLOAT); + reduction(dK_fullhead, + Reduction_attributes().set_name("red_dK_head").set_mode(ReductionMode_t::ADD), + attributes.outputs[output_names::dK]); + } + + auto const& amax_dk = attributes.outputs.at(output_names::Amax_dK); + if (amax_dk != nullptr) { + auto amax_dk_attributes = + Reduction_attributes().set_name("amax_dK").set_mode(ReductionMode_t::AMAX); + reduction(attributes.outputs[output_names::dK], amax_dk_attributes, amax_dk); + } + } else { + auto bmm_dP_T_Q_attributes = Matmul_fp8_attributes().set_name("bmm_dP_T_Q") + .set_m_override(attributes.inputs[input_names::SEQ_LEN_KV]) + .set_k_override(attributes.inputs[input_names::SEQ_LEN_Q]); + matmul_fp8(dP_T, + attributes.inputs[input_names::Q], + attributes.inputs[input_names::Descale_dP], + attributes.inputs[input_names::Descale_Q], + attributes.inputs[input_names::Scale_dK], + bmm_dP_T_Q_attributes, + attributes.outputs[output_names::dK], + attributes.outputs[output_names::Amax_dK]); + } + + return {error_code_t::OK, ""}; + } + + std::pair> + override_heuristics_query() const { + int32_t const sm_version = context.get_sm_version(); + bool const use_new_knobs = detail::get_backend_version() >= 92300; + // {128,128} bprop: tileM=3, tileN=2, kernelCfg=2(bprop warp), streamK=0, cgaM=0 + if (sm_version > 103 && is_deterministic_algorithm_supported_on_blackwell) { + if (use_new_knobs) { + return {17, + {{KnobType_t::TILE_M, 3}, + {KnobType_t::TILE_N, 2}, + {KnobType_t::KERNEL_CFG, 2}, + {KnobType_t::STREAM_K, 0}, + {KnobType_t::TILE_CGA_M, 0}, + {KnobType_t::STAGES, 2}}}; + } else { + return {17, {{KnobType_t::KERNEL_CFG, 31}, {KnobType_t::STAGES, 2}}}; + } + } else if (is_deterministic_algorithm_supported_on_blackwell) { + if (use_new_knobs) { + return {5, + {{KnobType_t::TILE_M, 3}, + {KnobType_t::TILE_N, 2}, + {KnobType_t::KERNEL_CFG, 2}, + {KnobType_t::STREAM_K, 0}, + {KnobType_t::TILE_CGA_M, 0}, + {KnobType_t::STAGES, 2}}}; + } else { + return {5, {{KnobType_t::KERNEL_CFG, 31}, {KnobType_t::STAGES, 2}}}; + } + } else { + return {-1, {}}; + } + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j["is_mxfp8"] = is_mxfp8_scaling(); + if (auto const rescale_threshold = get_rescale_threshold_from_env(); rescale_threshold.has_value()) { + j["rescale_threshold"] = rescale_threshold.value(); + } + if (is_mxfp8_scaling()) { + j.update(R"({"tag": "SDPA_MXFP8_BWD"})"_json); + } else { + j.update(R"({"tag": "SDPA_FP8_BWD"})"_json); + } + } +#endif +}; + +} // namespace cudnn_frontend::graph diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/sdpa_support_surface.h b/deps/cudnn-frontend/include/cudnn_frontend/node/sdpa_support_surface.h new file mode 100644 index 0000000..6b73f73 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/sdpa_support_surface.h @@ -0,0 +1,553 @@ +#pragma once + +#include +#include + +#include "../../cudnn_frontend_Heuristics.h" +#include "../../cudnn_frontend_Logging.h" +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend::graph { + +inline error_t +SDPA_attributes::validate_sdpa_support_surface(const detail::Context& context, + int64_t s_kv, + bool is_paged_k, + bool is_paged_v) const { + // Extract dimensions from tensors + int64_t s_q = inputs.at(SDPA_attributes::input_names::Q)->get_dim()[2]; + // s_kv is passed in from the caller + int64_t h_q = inputs.at(SDPA_attributes::input_names::Q)->get_dim()[1]; + int64_t h_k = inputs.at(SDPA_attributes::input_names::K)->get_dim()[1]; + int64_t h_v = inputs.at(SDPA_attributes::input_names::V)->get_dim()[1]; + int64_t d_qk = inputs.at(SDPA_attributes::input_names::Q)->get_dim()[3]; + int64_t d_v = inputs.at(SDPA_attributes::input_names::V)->get_dim()[3]; + + bool const is_ragged = inputs.at(SDPA_attributes::input_names::Q)->get_ragged_offset() || + inputs.at(SDPA_attributes::input_names::K)->get_ragged_offset() || + inputs.at(SDPA_attributes::input_names::V)->get_ragged_offset() || + outputs.at(SDPA_attributes::output_names::O)->get_ragged_offset(); + + auto const& output_tensor = outputs.at(SDPA_attributes::output_names::O); + auto const& output_data_type = output_tensor->get_data_type(); + + auto const& bias_mask = inputs.find(SDPA_attributes::input_names::Bias); + bool const is_bias = (bias_mask != inputs.end() && bias_mask->second != nullptr); + + auto const& dropout_mask = inputs.find(SDPA_attributes::input_names::Dropout_mask); + bool const is_dropout_custom = (dropout_mask != inputs.end()) && (dropout_mask->second != nullptr); + bool const is_dropout = dropout_probability.has_value() || is_dropout_custom; + + bool const is_paged = is_paged_k || is_paged_v; + + auto const& rng_tensor = outputs.find(SDPA_attributes::output_names::RNG_DUMP); + bool const is_rng = (rng_tensor != outputs.end() && rng_tensor->second != nullptr); + + bool const max_seq_kv_explicit = max_seq_len_kv.has_value(); + + auto const& attn_scale = inputs.find(SDPA_attributes::input_names::Attn_scale); + bool const has_attn_scale = (attn_scale != inputs.end()) && (attn_scale->second != nullptr); + + auto const& seq_len_q = inputs.find(SDPA_attributes::input_names::SEQ_LEN_Q); + bool const has_seq_len_q = (seq_len_q != inputs.end()) && (seq_len_q->second != nullptr); + auto const& seq_len_kv = inputs.find(SDPA_attributes::input_names::SEQ_LEN_KV); + bool const has_seq_len_kv = (seq_len_kv != inputs.end()) && (seq_len_kv->second != nullptr); + + auto const& cu_seq_len_q = inputs.find(SDPA_attributes::input_names::CU_SEQ_LEN_Q); + bool const has_cu_seq_len_q = (cu_seq_len_q != inputs.end()) && (cu_seq_len_q->second != nullptr); + auto const& cu_seq_len_kv = inputs.find(SDPA_attributes::input_names::CU_SEQ_LEN_KV); + bool const has_cu_seq_len_kv = (cu_seq_len_kv != inputs.end()) && (cu_seq_len_kv->second != nullptr); + + // validation TODO: + // - validate stats has valid dims + + // Get device SM version from context + CHECK_CUDNN_FRONTEND_ERROR(context.populate_sm_version_from_device()); + int32_t const sm_version = context.get_sm_version(); + int32_t const prop_major = sm_version / 10; + + // Common FP16 and FP8 validation + // validate basic dimension requirements + RETURN_CUDNN_FRONTEND_ERROR_IF( + (h_q % h_k != 0) || (h_q % h_v != 0), + error_code_t::GRAPH_NOT_SUPPORTED, + "For group-query attention, number of heads for key and query must be a factor of number of heads for query"); + + // validate options for attn_scale + RETURN_CUDNN_FRONTEND_ERROR_IF(has_attn_scale && attn_scale_value.has_value(), + error_code_t::ATTRIBUTE_NOT_SET, + "attn_scale with tensor and value cannot be set at the same time."); + + // validate options for bias mask + RETURN_CUDNN_FRONTEND_ERROR_IF(is_bias && (bias_mask->second->get_data_type() == DataType_t::BOOLEAN), + error_code_t::GRAPH_NOT_SUPPORTED, + "Bias mask data type cannot be boolean"); + + // validate options for padding mask: padding requires per-sequence length tensors, + // either as (SEQ_LEN_Q + SEQ_LEN_KV) or as (CU_SEQ_LEN_Q + CU_SEQ_LEN_KV). + RETURN_CUDNN_FRONTEND_ERROR_IF( + padding_mask && (!has_seq_len_q || !has_seq_len_kv) && (!has_cu_seq_len_q || !has_cu_seq_len_kv), + error_code_t::ATTRIBUTE_NOT_SET, + "Padding mask requires seq_len_q/seq_len_kv (or cu_seq_len_q/cu_seq_len_kv) to be set."); + RETURN_CUDNN_FRONTEND_ERROR_IF( + (!padding_mask && !attention_score_modifier) && + (has_seq_len_q || has_seq_len_kv || has_cu_seq_len_q || has_cu_seq_len_kv), + error_code_t::ATTRIBUTE_NOT_SET, + "seq_len_q/seq_len_kv (or cu_seq_len_q/cu_seq_len_kv) needs to be set only if padding mask is enabled."); + + // Cumulative sequence length tensors must be set together, and are mutually + // exclusive with the (per-batch) seq_len tensors. + RETURN_CUDNN_FRONTEND_ERROR_IF(has_cu_seq_len_q != has_cu_seq_len_kv, + error_code_t::ATTRIBUTE_NOT_SET, + "cu_seq_len_q and cu_seq_len_kv must both be set or both unset."); + RETURN_CUDNN_FRONTEND_ERROR_IF((has_cu_seq_len_q || has_cu_seq_len_kv) && (has_seq_len_q || has_seq_len_kv), + error_code_t::INVALID_VALUE, + "Cannot specify both seq_len tensors and cu_seq_len tensors."); + + RETURN_CUDNN_FRONTEND_ERROR_IF(is_ragged && ((padding_mask == false) && (attention_score_modifier == nullptr)), + error_code_t::GRAPH_NOT_SUPPORTED, + "Ragged offsets are only supported with padding mask."); + + // validate options for dropout mask + RETURN_CUDNN_FRONTEND_ERROR_IF( + dropout_probability.has_value() && is_dropout_custom, + error_code_t::ATTRIBUTE_NOT_SET, + "Using both, custom dropout mask and internal-mask generation using dropout probability, is ill-formed."); + + RETURN_CUDNN_FRONTEND_ERROR_IF(dropout_probability.has_value() && dropout_probability.value() == 1.0, + error_code_t::ATTRIBUTE_NOT_SET, + "Dropout probability cannot be 1 as corresponding scale wont be well formed."); + + // validate options for causal mask and bottom right causal mask + RETURN_CUDNN_FRONTEND_ERROR_IF( + has_causal_mask_bottom_right() && (!padding_mask) && s_q > s_kv, + error_code_t::GRAPH_NOT_SUPPORTED, + "Bottom right causal mask does not support max_s_q > max_s_kv. Please virtually slice the Q tensor and pass it " + "as max_s_q == max_s_kv"); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + has_causal_mask_bottom_right() && (is_bias || alibi_mask || is_dropout), + error_code_t::GRAPH_NOT_SUPPORTED, + "Bottom right causal mask is only supported with is_bias=False, is_alibi=False, is_dropout=False."); + + RETURN_CUDNN_FRONTEND_ERROR_IF(has_causal_mask_bottom_right() && (detail::get_backend_version() < 90600) && + ((s_q % 64 != 0) || (s_kv % 64 != 0)), + error_code_t::GRAPH_NOT_SUPPORTED, + "Bottom right causal mask is only supported with s_q multiple of 64, and s_kv " + "multiple of 64, for cudnn version below 9.6.0"); + + RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 90600 && left_bound.has_value() && + has_causal_mask_bottom_right() && s_q != s_kv, + error_code_t::GRAPH_NOT_SUPPORTED, + "Sliding window attention with bottom right causal mask requires s_q == s_kv " + "for cuDNN version below 9.6.0"); + + // validate that datatype is set for the graph + RETURN_CUDNN_FRONTEND_ERROR_IF(context.get_intermediate_data_type() == DataType_t::NOT_SET, + error_code_t::ATTRIBUTE_NOT_SET, + "Intermediate tensor data type needs to be set as internal tensors require it."); + + if (mma_core_mode == DataType_t::FP8_E4M3 || mma_core_mode == DataType_t::FP8_E5M2) { + // FP8 specific validation + + RETURN_CUDNN_FRONTEND_ERROR_IF((prop_major == 12) && is_ragged, + error_code_t::GRAPH_NOT_SUPPORTED, + "sdpa fp8 with THD not supported for sm120 yet."); + + // version specific validation + RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 90100, + error_code_t::GRAPH_NOT_SUPPORTED, + "sdpa fp8 forward operation is only supported starting cudnn 9.1.0. Please " + "consider upgrading your current version."); + RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() == 91000, + error_code_t::GRAPH_NOT_SUPPORTED, + "sdpa fp8 forward operation is not supported on cudnn 9.10.0. Please " + "consider upgrading your current version."); + RETURN_CUDNN_FRONTEND_ERROR_IF( + prop_major < 9, + error_code_t::GRAPH_NOT_SUPPORTED, + "sdpa fp8 forward operation is only supported on Hopper architecture and newer. Please " + "consider using a newer architecture."); + + // FP8 does not support bias (TE constraint) + RETURN_CUDNN_FRONTEND_ERROR_IF(is_bias, error_code_t::GRAPH_NOT_SUPPORTED, "SDPA FP8 does not support bias"); + + // validate basic dimension requirements + // d_qk=192 with d_v=128 is only supported starting from cuDNN 9.19 + bool const d192_v128_supported = (detail::get_backend_version() >= 91900); + if (prop_major >= 10) { + RETURN_CUDNN_FRONTEND_ERROR_IF( + ((d_qk > 128) || (d_qk % 16 != 0)) && !(d192_v128_supported && d_qk == 192 && d_v == 128), + error_code_t::GRAPH_NOT_SUPPORTED, + "hidden_dim d_qk should be less than or equal to 128 and hidden_dim d_qk " + "should be multiple of 16 unless d_qk == 192 and d_v == 128 (requires cuDNN 9.19+)"); + RETURN_CUDNN_FRONTEND_ERROR_IF( + ((d_v > 128) || (d_v % 16 != 0)), + error_code_t::GRAPH_NOT_SUPPORTED, + "hidden_dim d_v should be less than or equal to 128 and hidden_dim d_v should be multiple of 16"); + } else { + RETURN_CUDNN_FRONTEND_ERROR_IF( + (d_qk > 256) || (d_qk % 16 != 0) || (d_v > 256) || (d_v % 16 != 0), + error_code_t::GRAPH_NOT_SUPPORTED, + "hidden_dim shoud be less than or equal to 256 and hidden_dim should be multiple of 16"); + } + + // Validate options for causal_mask_bottom_right + RETURN_CUDNN_FRONTEND_ERROR_IF(has_causal_mask_bottom_right() && detail::get_backend_version() < 90700, + error_code_t::GRAPH_NOT_SUPPORTED, + "For cuDNN version below 9.7.0, bottom right causal masking is not supported."); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + has_causal_mask_bottom_right() && prop_major < 10, + error_code_t::GRAPH_NOT_SUPPORTED, + "sdpa fp8 forward operation is only supported on Blackwell architecture and newer. Please " + "consider using a newer architecture."); + + // if output data type is half or bfloat16, and version is below 9.13 or is not blackwell, return NOT_SUPPORTED + RETURN_CUDNN_FRONTEND_ERROR_IF( + (output_data_type == DataType_t::HALF || output_data_type == DataType_t::BFLOAT16) && + (detail::get_backend_version() < 91300 || prop_major < 10), + error_code_t::GRAPH_NOT_SUPPORTED, + "sdpa fp8 forward with HALF/BFLOAT16 output is only supported on Blackwell architecture " + "with cuDNN version 9.13.0 and newer."); + } else if (mma_core_mode == DataType_t::HALF) { + // FP16 specific validation + + // Bug workarounds for known problematic versions + RETURN_CUDNN_FRONTEND_ERROR_IF( + detail::get_backend_version() == 91000 || detail::get_backend_version() == 91001, + error_code_t::GRAPH_NOT_SUPPORTED, + "SDPA FP16/BF16 forward is not supported on cuDNN 9.10.0/9.10.1 due to known bugs. " + "Please consider upgrading to 9.10.2 or newer."); + + // 9.14.0 sliding window bug: non-causal + s_kv > 1024 + sliding window + RETURN_CUDNN_FRONTEND_ERROR_IF( + detail::get_backend_version() == 91400 && s_kv > 1024 && left_bound.has_value() && + !has_causal_like_masking(), + error_code_t::GRAPH_NOT_SUPPORTED, + "cuDNN 9.14.0 has a known bug with non-causal + s_kv > 1024 + sliding window attention. " + "Please consider upgrading to 9.14.1 or newer."); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + (attention_score_modifier != nullptr) && + (alibi_mask || has_causal_like_masking() || padding_mask || left_bound.has_value()), + error_code_t::GRAPH_NOT_SUPPORTED, + "Attention score mod enabled and hence other subgraphs are disabled."); + + // validate basic dimension requirements + RETURN_CUDNN_FRONTEND_ERROR_IF( + (d_qk % 8 != 0) || (d_v % 8 != 0), error_code_t::GRAPH_NOT_SUPPORTED, "hidden_dim should be multiple of 8"); + + // validate alibi requirements + RETURN_CUDNN_FRONTEND_ERROR_IF(alibi_mask && !(right_bound.has_value() && right_bound.value() == 0), + error_code_t::GRAPH_NOT_SUPPORTED, + "When alibi mask is used, diagonal_band_right_bound needs to be set to 0."); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + left_bound.has_value() && left_bound.value() <= 0 && detail::get_backend_version() < 91000, + error_code_t::INVALID_VALUE, + "Left bound (Sliding window length) should be greater than zero when set."); + + RETURN_CUDNN_FRONTEND_ERROR_IF(left_bound.has_value() && (!padding_mask) && s_q > s_kv, + error_code_t::GRAPH_NOT_SUPPORTED, + "Sliding window attention is only supported with max_s_q <= max_s_kv."); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + left_bound.has_value() && (s_q * left_bound.value() == s_kv * left_bound.value()) && + (detail::get_backend_version() <= 90900) && (prop_major == 9) && has_causal_mask_bottom_right(), + error_code_t::GRAPH_NOT_SUPPORTED, + "On Hopper architecture, this specific combination of s_q, s_kv, and left_bound + right_bound + bottom " + "right diagonal alignment is not supported for backend version 9.9 or below"); + + if ((detail::get_backend_version() < 91002)) { + RETURN_CUDNN_FRONTEND_ERROR_IF( + left_bound.has_value() && (!has_causal_like_masking() || is_dropout || is_bias), + error_code_t::GRAPH_NOT_SUPPORTED, + "Left and right bounds are only supported with is_dropout=False, is_bias=False. And the diagonal " + "alignment must be set."); + } + + RETURN_CUDNN_FRONTEND_ERROR_IF(right_bound.has_value() && right_bound.value() < 0, + error_code_t::INVALID_VALUE, + "Right bound needs to be larger than or equal to zero"); + + // Validate options for s_q == 1 + const bool is_decode_only = (s_q == 1); + RETURN_CUDNN_FRONTEND_ERROR_IF(is_decode_only && (prop_major == 10) && (d_qk > 128 || d_v > 128) && + (detail::get_backend_version() <= 90900), + error_code_t::GRAPH_NOT_SUPPORTED, + "decode only mode, i.e. s_q == 1 not supported for blackwell architecture with " + "d_qk or d_v > 128 for backend version 9.9 or below"); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + is_decode_only && (detail::get_backend_version() <= 90900) && (right_bound.has_value()), + error_code_t::GRAPH_NOT_SUPPORTED, + "decode only mode, i.e. s_q == 1, not supported with masking (right_bound is set) for backend version 9.9 " + "or below"); + + RETURN_CUDNN_FRONTEND_ERROR_IF(is_decode_only && has_sink_token(), + error_code_t::GRAPH_NOT_SUPPORTED, + "decode only mode, i.e. s_q == 1, not supported with sink_token"); + + // validate options for paged attention + RETURN_CUDNN_FRONTEND_ERROR_IF( + is_paged && (d_qk > 128 || d_v > 128) && detail::get_backend_version() <= 90900, + error_code_t::GRAPH_NOT_SUPPORTED, + "Paged attention only supported with d_qk and d_v <= 128 for backend version 9.9 or below"); + + RETURN_CUDNN_FRONTEND_ERROR_IF(is_paged && is_ragged && detail::get_backend_version() < 90700, + error_code_t::GRAPH_NOT_SUPPORTED, + "Paged caches are not supported in combination with ragged offsets."); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + is_paged && !((has_seq_len_q && has_seq_len_kv) || (has_cu_seq_len_q && has_cu_seq_len_kv)), + error_code_t::GRAPH_NOT_SUPPORTED, + "Paged caches can only be used in combination with padding mask and variable sequence lengths " + "for both Q and KV (via seq_len_q/seq_len_kv or cu_seq_len_q/cu_seq_len_kv)."); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + !is_paged && max_seq_kv_explicit, + error_code_t::GRAPH_NOT_SUPPORTED, + "When not using paged attention, there is no need to explicitly set max kv sequence length."); + + if (max_seq_kv_explicit) { + auto max_seq_kv = max_seq_len_kv.value(); + + RETURN_CUDNN_FRONTEND_ERROR_IF(is_bias && (bias_mask->second->get_dim()[3] != max_seq_kv), + error_code_t::GRAPH_NOT_SUPPORTED, + "Value set through set_paged_attention_max_seq_len_kv is incompatible with " + "the sequence length of the bias"); + + RETURN_CUDNN_FRONTEND_ERROR_IF(is_rng && rng_tensor->second->get_dim()[3] != max_seq_kv, + error_code_t::GRAPH_NOT_SUPPORTED, + "Value set through set_paged_attention_max_seq_len_kv is incompatible with " + "the sequence length of the RNG_DUMP"); + } + + // Additional validation for paged attention with packed page tables + RETURN_CUDNN_FRONTEND_ERROR_IF( + ((is_paged_k && inputs.at(SDPA_attributes::input_names::Page_table_K)->get_ragged_offset()) || + (is_paged_v && inputs.at(SDPA_attributes::input_names::Page_table_V)->get_ragged_offset())) && + detail::get_backend_version() < 91002, + error_code_t::GRAPH_NOT_SUPPORTED, + "Paged attention with packed page tables only supported with cudnn version 9.10.2 and above"); + + // SM version check for SDPA + RETURN_CUDNN_FRONTEND_ERROR_IF(prop_major < 8, + error_code_t::GRAPH_NOT_SUPPORTED, + "SDPA FP16/BF16 requires SM80 (Ampere) or newer architecture"); + + // version specific validation by architecture + if (prop_major == 8) { + RETURN_CUDNN_FRONTEND_ERROR_IF( + detail::get_backend_version() <= 90900 && ((d_qk > 128) || (d_v > 128)), + error_code_t::GRAPH_NOT_SUPPORTED, + "head_dim should be less than or equal to 128 for backend version 9.9 or below on ampere architecture"); + } + if (prop_major == 9) { + RETURN_CUDNN_FRONTEND_ERROR_IF( + detail::get_backend_version() <= 90900 && ((d_qk > 256) || (d_v > 256)), + error_code_t::GRAPH_NOT_SUPPORTED, + "head_dim should be less than or equal to 256 for backend version 9.9 or below on hopper architecture"); + } + if (prop_major == 10) { + RETURN_CUDNN_FRONTEND_ERROR_IF((detail::get_backend_version() < 90900) && ((d_qk > 128) || (d_v > 128)), + error_code_t::GRAPH_NOT_SUPPORTED, + "head_dim should be less than or equal to 128 for backend version 9.8 or " + "below on blackwell architecture"); + } + + RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 90500 && is_paged, + error_code_t::GRAPH_NOT_SUPPORTED, + "For cuDNN version below 9.5.0, paged caches are not supported"); + + if (is_ragged) { + RETURN_CUDNN_FRONTEND_ERROR_IF((context.get_sm_version() > 0 && context.get_sm_version() < 90 && + detail::get_backend_version() < 91801), + error_code_t::GRAPH_NOT_SUPPORTED, + "THD (ragged offset) is only supported in Hopper and above : " + + std::to_string(context.get_sm_version())); + } + } else { + RETURN_CUDNN_FRONTEND_ERROR_IF(true, error_code_t::GRAPH_NOT_SUPPORTED, "Unsupported mma core mode"); + } + + // Check whether the selected implementation supports the requested features. + CHECK_CUDNN_FRONTEND_ERROR(verify_sdpa_support_surface_for_implementation(context, implementation)); + + return {error_code_t::OK, ""}; +} + +// Verify that the underlying implementation supports all the features in these attributes. +// Unlike `validate_sdpa_support_surface()`, this may be called before validation, so: +// * don't assume any particular keys already exist in `inputs` or `outputs` +// * don't assume any tensor dims or strides are already set +// We return error codes directly instead of using `RETURN_CUDNN_FRONTEND_ERROR_IF` +// to avoid unneeded logging when this function is being called in a non-error-generating +// situation (e.g. during auto-select of SDPA implementation). +inline error_t +SDPA_attributes::verify_sdpa_support_surface_for_implementation(const detail::Context& context, + AttentionImplementation_t impl) const { + switch (impl) { + case AttentionImplementation_t::AUTO: + // This function should not be called with AUTO. + return {error_code_t::INVALID_VALUE, + "Can't call verify_sdpa_support_surface_for_implementation with impl=AUTO"}; + case AttentionImplementation_t::COMPOSITE: + for (const auto& [key, value] : inputs) { + RETURN_CUDNN_FRONTEND_ERROR_IF(key == input_names::Block_mask && value != nullptr, + error_code_t::GRAPH_NOT_SUPPORTED, + "Composite SDPA node doesn't support Block_mask input"); + RETURN_CUDNN_FRONTEND_ERROR_IF( + (key == input_names::CU_SEQ_LEN_Q || key == input_names::CU_SEQ_LEN_KV) && value != nullptr, + error_code_t::GRAPH_NOT_SUPPORTED, + "Composite SDPA node doesn't support CU_SEQ_LEN_Q / CU_SEQ_LEN_KV inputs"); + } + // The ragged offset multiplier is only supported by the unified forward engine. + // Reject it here so auto-select routes such graphs to the unified implementation. + for (const auto& [key, value] : inputs) { + RETURN_CUDNN_FRONTEND_ERROR_IF(value != nullptr && value->has_ragged_offset_multiplier(), + error_code_t::GRAPH_NOT_SUPPORTED, + "Composite SDPA node doesn't support a ragged offset multiplier"); + } + for (const auto& [key, value] : outputs) { + RETURN_CUDNN_FRONTEND_ERROR_IF(value != nullptr && value->has_ragged_offset_multiplier(), + error_code_t::GRAPH_NOT_SUPPORTED, + "Composite SDPA node doesn't support a ragged offset multiplier"); + } + break; + case AttentionImplementation_t::UNIFIED: { + auto effective_cudnn_ver = std::min(detail::get_backend_version(), detail::get_compiled_version()); + RETURN_CUDNN_FRONTEND_ERROR_IF(effective_cudnn_ver < 91301, + error_code_t::GRAPH_NOT_SUPPORTED, + "Unified SDPA node requires cuDNN 9.13.1"); + + RETURN_CUDNN_FRONTEND_ERROR_IF(context.get_dynamic_shape_enabled(), + error_code_t::GRAPH_NOT_SUPPORTED, + "Unified SDPA node doesn't yet support dynamic shape"); + + // TODO: Provide smarter error messages that provide the required cuDNN version for each input. + std::unordered_set allowed_input_names{ + input_names::Q, input_names::K, input_names::V, input_names::Attn_scale}; + std::string allowed_input_msg = + "Unified SDPA node doesn't yet support inputs other than Q, K, V, Attn_scale"; + + if (effective_cudnn_ver >= 91400) { + allowed_input_names.insert({input_names::Block_mask}); + allowed_input_msg += ", Block_mask"; + } + + if (effective_cudnn_ver >= 91500) { + allowed_input_names.insert({input_names::Page_table_K, + input_names::Page_table_V, + input_names::SEQ_LEN_Q, + input_names::SEQ_LEN_KV}); + allowed_input_msg += ", Page_table_K, Page_table_V, SEQ_LEN_Q, SEQ_LEN_KV"; + } + + if (effective_cudnn_ver >= 92100) { + // NOTE: For unified engine, we support dropout via Seed and Offset only. + // Custom dropout mask (via Dropout_mask, Dropout_scale) is not supported. + allowed_input_names.insert( + {input_names::Bias, input_names::Seed, input_names::Offset, input_names::SINK_TOKEN}); + allowed_input_msg += ", Bias, Seed, Offset, SINK_TOKEN"; + } + + if (effective_cudnn_ver >= 92400) { + allowed_input_names.insert({input_names::CU_SEQ_LEN_Q, input_names::CU_SEQ_LEN_KV}); + allowed_input_msg += ", CU_SEQ_LEN_Q, CU_SEQ_LEN_KV"; + } + + if (effective_cudnn_ver >= 93000) { + allowed_input_names.insert({input_names::Descale_Q, + input_names::Descale_K, + input_names::Descale_V, + input_names::Descale_S, + input_names::Scale_S, + input_names::Scale_O}); + allowed_input_msg += ", Descale_Q, Descale_K, Descale_V, Descale_S, Scale_S, Scale_O"; + } + + for (const auto& [key, value] : inputs) { + if (allowed_input_names.find(key) == allowed_input_names.end() && value != nullptr) { + return {error_code_t::GRAPH_NOT_SUPPORTED, allowed_input_msg}; + } + } + + std::unordered_set allowed_output_names{output_names::O, + output_names::Stats}; + std::string allowed_output_msg = "Unified SDPA node doesn't yet support outputs other than O, Stats"; + + if (effective_cudnn_ver >= 92100) { + allowed_output_names.insert({output_names::RNG_DUMP, output_names::Max, output_names::Sum_exp}); + allowed_output_msg += ", RNG_DUMP, Max, Sum_exp"; + } + + if (effective_cudnn_ver >= 93000) { + allowed_output_names.insert({output_names::Amax_S, output_names::Amax_O}); + allowed_output_msg += ", Amax_S, Amax_O"; + } + + for (const auto& [key, value] : outputs) { + if (allowed_output_names.find(key) == allowed_output_names.end() && value != nullptr) { + return {error_code_t::GRAPH_NOT_SUPPORTED, allowed_output_msg}; + } + } + + if (alibi_mask && effective_cudnn_ver < 92100) { + return {error_code_t::GRAPH_NOT_SUPPORTED, + "Alibi mask for unified SDPA node requires cuDNN 9.21.0 or above"}; + } + + if (padding_mask && effective_cudnn_ver < 91500) { + return {error_code_t::GRAPH_NOT_SUPPORTED, + "Padding mask for unified SDPA node requires cuDNN 9.15.0 or above"}; + } + + if ((left_bound.has_value() || right_bound.has_value()) && effective_cudnn_ver < 92100) { + return {error_code_t::GRAPH_NOT_SUPPORTED, + "Left bound or right bound for unified SDPA node requires cuDNN 9.21.0 or above"}; + } + + if (diagonal_alignment != DiagonalAlignment_t::TOP_LEFT && effective_cudnn_ver < 92100) { + return {error_code_t::GRAPH_NOT_SUPPORTED, + "Diagonal alignment for unified SDPA node requires cuDNN 9.21.0 or above"}; + } + + if (dropout_probability.has_value() && effective_cudnn_ver < 92200) { + return {error_code_t::GRAPH_NOT_SUPPORTED, "Dropout for unified SDPA node requires cuDNN 9.22.0"}; + } + + if (dropout_probability.has_value() && generate_stats.value_or(false)) { + return {error_code_t::GRAPH_NOT_SUPPORTED, + "Dropout for unified SDPA node with generated stats is not supported"}; + } + + // Unified engine in cuDNN < 9.15 can't meaningfully support max sequence length, + // while versions >= 9.15 "support" it by ignoring it (unified engine doesn't need it). + if (max_seq_len_kv.has_value() && effective_cudnn_ver < 91500) { + return {error_code_t::GRAPH_NOT_SUPPORTED, + "Max sequence length for unified SDPA node cannot be set in cuDNN < 9.15.0"}; + } + + if (attention_score_modifier != nullptr && effective_cudnn_ver < 92100) { + return {error_code_t::GRAPH_NOT_SUPPORTED, + "Attention score modifier for unified SDPA node requires cuDNN 9.21.0 or above"}; + } + + if (mma_core_mode == DataType_t::FP8_E4M3 || mma_core_mode == DataType_t::FP8_E5M2) { + // Per-tensor FP8 and MXFP8 (block-scaled, E8M0 descales) are supported by the + // unified node starting from cuDNN 9.30.0. + if (effective_cudnn_ver < 93000) { + return {error_code_t::GRAPH_NOT_SUPPORTED, + "FP8/MXFP8 for the unified SDPA node requires cuDNN 9.30.0 or above"}; + } + } + } break; + } + + return {error_code_t::OK, ""}; +} + +} // namespace cudnn_frontend::graph diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/slice.h b/deps/cudnn-frontend/include/cudnn_frontend/node/slice.h new file mode 100644 index 0000000..a1aafb1 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/slice.h @@ -0,0 +1,224 @@ +#pragma once + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend::graph { + +class SliceNode : public NodeCRTP { + public: + Slice_attributes attributes; + + SliceNode(Slice_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::SLICE; + } + + error_t + infer_properties_node() override final { + getLogger() << "[cudnn_frontend] INFO: Inferrencing properties for slice node " << attributes.name + << std::endl; + + attributes.fill_from_context(context); + + for (size_t i = 0; i < attributes.slice_strides.size(); ++i) { + RETURN_CUDNN_FRONTEND_ERROR_IF( + attributes.slice_strides[i] <= 0, + error_code_t::INVALID_VALUE, + "Slice slice_strides[" + std::to_string(i) + "] must be strictly positive (got " + + std::to_string(attributes.slice_strides[i]) + + "). Non-positive strides break output dimension calculation and can cause division by zero."); + } + + auto output = attributes.outputs.at(Slice_attributes::output_names::Y); + auto output_dim = output->get_dim(); + + if (output_dim.empty()) { + for (size_t i = 0; i < attributes.slices.size(); ++i) { + int64_t start = attributes.slices[i].first; + int64_t limit = attributes.slices[i].second; + int64_t stride = (!attributes.slice_strides.empty() && i < attributes.slice_strides.size()) + ? attributes.slice_strides[i] + : 1; + // Output dimension = ceil((limit - start) / stride) + int64_t dim = (limit - start + stride - 1) / stride; + output_dim.push_back(dim); + } + output->set_dim(output_dim); + } + + auto const input = attributes.inputs.at(Slice_attributes::input_names::X); + auto const input_data_type = input->get_data_type(); + auto const output_data_type = output->get_data_type(); + if (output_data_type == DataType_t::NOT_SET) { + output->set_data_type(input_data_type); + } else { + RETURN_CUDNN_FRONTEND_ERROR_IF(output_data_type != input_data_type, + error_code_t::INVALID_VALUE, + "output and input tensor data types should match for slice operation."); + } + + auto const input_stride = input->get_stride(); + if (output->get_stride().empty()) { + // When slice strides > 1, output strides need to be multiplied accordingly + std::vector output_stride; + for (size_t i = 0; i < input_stride.size(); ++i) { + int64_t stride = (!attributes.slice_strides.empty() && i < attributes.slice_strides.size()) + ? attributes.slice_strides[i] + : 1; + output_stride.push_back(input_stride[i] * stride); + } + output->set_stride(output_stride); + } + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_tensors_node(std::unordered_map>& tensors, + int64_t& potential_uid, + std::unordered_set const& used_uids) const override final { + getLogger() << "[cudnn_frontend] INFO: Creating cudnn tensors for SliceNode " << attributes.name << std::endl; + + auto const input = attributes.inputs.at(Slice_attributes::input_names::X); + auto const output = attributes.outputs.at(Slice_attributes::output_names::Y); + + if (detail::get_backend_version() >= 92200 && detail::get_compiled_version() >= 92200) { + CHECK_CUDNN_FRONTEND_ERROR(detail::create_cudnn_tensor(input, tensors, potential_uid, used_uids)); + CHECK_CUDNN_FRONTEND_ERROR(detail::create_cudnn_tensor(output, tensors, potential_uid, used_uids)); + return {error_code_t::OK, ""}; + } + + if (input->has_uid() == false) { + detail::assign_uid(input.get(), potential_uid, used_uids); + } + output->set_is_virtual(false); + CHECK_CUDNN_FRONTEND_ERROR(detail::create_cudnn_tensor(output, tensors, potential_uid, used_uids)); + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set& uids_involved_in_operations, + std::vector>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map>& tensors) const override final { + getLogger() << "[cudnn_frontend] INFO: " << "Building SliceNode operations " << attributes.name << std::endl; + +#if (CUDNN_VERSION >= 92200) + // cuDNN >= 9.22.0: Use native backend slice operation + auto cudnn_ver_error = error_t{error_code_t::GRAPH_NOT_SUPPORTED, "Slice backend operation requires 9.22.0"}; + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(92200, cudnn_ver_error); + + if (detail::get_backend_version() >= 92200) { + CUDNN_FRONTEND_UNUSED(operations); + + auto slice_operation = make_shared_backend_pointer(CUDNN_BACKEND_OPERATION_SLICE_DESCRIPTOR); + + // Set input tensor + auto X = attributes.inputs.at(Slice_attributes::input_names::X); + auto backend_x = tensors[X->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(slice_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SLICE_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_x)); + + // Set output tensor + auto Y = attributes.outputs.at(Slice_attributes::output_names::Y); + auto backend_y = tensors[Y->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(slice_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SLICE_YDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_y)); + + // Extract start and limit indices from slices + std::vector start_indices; + std::vector limit_indices; + + for (const auto& slice : attributes.slices) { + start_indices.push_back(slice.first); + limit_indices.push_back(slice.second); + } + + // Per-dimension strides: use user slice_strides[i] when set, else 1 (preserves partial configuration) + std::vector strides(attributes.slices.size()); + for (size_t i = 0; i < strides.size(); ++i) { + strides[i] = (i < attributes.slice_strides.size()) ? attributes.slice_strides[i] : 1; + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(slice_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SLICE_START_INDICES, + CUDNN_TYPE_INT64, + start_indices.size(), + start_indices.data())); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(slice_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SLICE_LIMIT_INDICES, + CUDNN_TYPE_INT64, + limit_indices.size(), + limit_indices.data())); + + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(slice_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SLICE_STRIDES, + CUDNN_TYPE_INT64, + strides.size(), + strides.data())); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(slice_operation->get_backend_descriptor())); + + raw_operations.push_back(slice_operation); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; + } +#endif + CUDNN_FRONTEND_UNUSED(operations); + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FRONTEND_UNUSED(tensors); + + getLogger() << "[cudnn_frontend] INFO: " << "Using pointer arithmetic fallback for slice on cuDNN < 9.22.0" + << std::endl; + + auto const output = attributes.outputs.at(Slice_attributes::output_names::Y); + if (output && output->get_is_virtual() == false) { + uids_involved_in_operations.insert(output->get_uid()); + if (auto ragged_offset = output->get_ragged_offset()) { + uids_involved_in_operations.insert(ragged_offset->get_uid()); + } + } + return {error_code_t::OK, ""}; + } + + error_t + collect_variant_pack_replacements_node( + std::unordered_map>& + variant_pack_replacements) const override final { + if (detail::get_backend_version() >= 92200 && detail::get_compiled_version() >= 92200) { + CUDNN_FRONTEND_UNUSED(variant_pack_replacements); + return {error_code_t::OK, ""}; + } + auto const input = attributes.inputs.at(Slice_attributes::input_names::X); + auto const output = attributes.outputs.at(Slice_attributes::output_names::Y); + + variant_pack_replacements[input->get_uid()] = {output->get_uid(), attributes.get_offset()}; + + return {error_code_t::OK, ""}; + }; + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "SLICE"})"_json); + } +#endif +}; + +} // namespace cudnn_frontend::graph \ No newline at end of file diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/softmax.h b/deps/cudnn-frontend/include/cudnn_frontend/node/softmax.h new file mode 100644 index 0000000..1aa5689 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/softmax.h @@ -0,0 +1,366 @@ +#pragma once + +#include "../../cudnn_frontend_Heuristics.h" +#include "../../cudnn_frontend_Logging.h" + +#include "../graph_helpers.h" +#include "../node_interface.h" + +#include "pointwise.h" +#include "reduction.h" + +namespace cudnn_frontend::graph { + +// Base class for nodes that represent a softmax operation. +// This is either implemented as a composite of pointwise/reduction operations (CompositeSoftmaxNode) +// or as a single unified backend operation (UnifiedSoftmaxNode). +template +class SoftmaxNodeBase : public NodeCRTP { + protected: + using input_names = Softmax_attributes::input_names; + using output_names = Softmax_attributes::output_names; + + public: + Softmax_attributes attributes; + + SoftmaxNodeBase(Softmax_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + error_t + infer_properties_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for Softmax node " << attributes.name); + + attributes.fill_from_context(this->context); + + auto input_tensor = attributes.inputs.at(Softmax_attributes::input_names::P); + auto output_tensor = attributes.outputs.at(Softmax_attributes::output_names::S); + + // Only infer dims and strides if user did not set them + if (output_tensor->get_dim().empty()) { + output_tensor->set_dim(input_tensor->get_dim()); + } + + if (output_tensor->get_stride().empty()) { + output_tensor->set_stride(input_tensor->get_stride()); + } + + return {error_code_t::OK, ""}; + } + + protected: + bool + has_sink() const { + auto sink_it = attributes.inputs.find(Softmax_attributes::input_names::SINK); + return ((sink_it) != attributes.inputs.end() && sink_it->second != nullptr); + } + + bool + has_stats() const { + auto stats_it = attributes.outputs.find(Softmax_attributes::output_names::Stats); + return ((stats_it) != attributes.outputs.end() && stats_it->second != nullptr); + } + + bool + has_max() const { + auto max_it = attributes.outputs.find(Softmax_attributes::output_names::Max); + return ((max_it) != attributes.outputs.end() && max_it->second != nullptr); + } + + bool + has_sum_exp() const { + auto sum_exp_it = attributes.outputs.find(Softmax_attributes::output_names::Sum_exp); + return ((sum_exp_it) != attributes.outputs.end() && sum_exp_it->second != nullptr); + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + void + serialize(json& j) const override final { + j = attributes; + } +#endif +}; + +// Fallback implementation of softmax node that represents the operation as a series of pointwise and reduction +// operations. This is used for cuDNN versions before v9.21.0. +// Only certain combinations of outputs are allowed. +class CompositeSoftmaxNode : public SoftmaxNodeBase { + public: + CompositeSoftmaxNode(Softmax_attributes&& attributes_, detail::Context const& context) + : SoftmaxNodeBase(std::move(attributes_), context) {} + + Type + getType() override final { + return Type::COMPOSITE; + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating CompositeSoftmaxNode " << attributes.name); + + RETURN_CUDNN_FRONTEND_ERROR_IF( + !((!has_stats() && !has_max() && !has_sum_exp()) || // + (has_stats() && !has_max() && !has_sum_exp()) || // + (!has_stats() && has_max() && has_sum_exp())), + error_code_t::GRAPH_NOT_SUPPORTED, + "CompositeSoftmaxNode can only output certain combinations of stats, max and sum_exp: " + "stats only, max and sum_exp only, or none of the above."); + + return {error_code_t::OK, ""}; + } + + error_t + expand_node() override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Inferrencing properties for CompositeSoftmaxNode " << attributes.name); + + attributes.fill_from_context(context); + + // Fill properties of virtual tensors + auto const p_dim = attributes.inputs[Softmax_attributes::input_names::P]->get_dim(); + auto b = p_dim[0]; + auto h = p_dim[1]; + auto s_q = p_dim[2]; + + std::shared_ptr max_output; + if (has_max()) { + max_output = attributes.outputs[Softmax_attributes::output_names::Max]; + } else { + max_output = std::make_shared(); + max_output->set_is_virtual(true); + } + //////////////// TODO ////////////////////////// + // Check Stride (Before setting dimension?) + if (max_output->get_dim().empty()) { + max_output->set_dim({b, h, s_q, 1}); + } + if (max_output->get_stride().empty()) { + max_output->set_stride({h * s_q, s_q, 1, 1}); + } + + auto max_attributes = Reduction_attributes().set_name("Max").set_mode(ReductionMode_t::MAX); + // If sink tensor is present, we also need to take a pointwise max with sink + if (has_sink()) { + auto s_max = reduction(attributes.inputs[Softmax_attributes::input_names::P], max_attributes); + s_max->set_name("s_max"); + + auto sink_tensor = attributes.inputs[Softmax_attributes::input_names::SINK]; + auto sink_attributes = Pointwise_attributes().set_name("max_sink").set_mode(PointwiseMode_t::MAX); + pointwise(s_max, sink_tensor, sink_attributes, max_output); + } else { + // Special non-functional-style call. Needed because output already created and provided to user. + reduction(attributes.inputs[Softmax_attributes::input_names::P], max_attributes, max_output); + } + + auto sub_attributes = Pointwise_attributes().set_name("sub").set_mode(PointwiseMode_t::SUB); + auto const& sub_output = + pointwise(attributes.inputs[Softmax_attributes::input_names::P], max_output, sub_attributes); + sub_output->set_name("sub_M"); + + auto exp_attributes = Pointwise_attributes().set_name("exp").set_mode(PointwiseMode_t::EXP); + auto const& exp_output = pointwise(sub_output, exp_attributes); + exp_output->set_name("exp_sub_M"); + + std::shared_ptr sum_output; + if (has_sum_exp()) { + sum_output = attributes.outputs[Softmax_attributes::output_names::Sum_exp]; + } else { + sum_output = std::make_shared(); + sum_output->set_is_virtual(true); + } + sum_output->set_name("SumExp"); + if (sum_output->get_dim().empty()) { + sum_output->set_dim({b, h, s_q, 1}); + } + if (sum_output->get_stride().empty()) { + sum_output->set_stride({h * s_q, s_q, 1, 1}); + } + auto sum_attributes = Reduction_attributes().set_name("sum").set_mode(ReductionMode_t::ADD); + // If sink tensor is present, also subtract it and take its exp + if (has_sink()) { + auto sink_tensor = attributes.inputs[Softmax_attributes::input_names::SINK]; + auto sub_sink = pointwise(sink_tensor, max_output, sub_attributes); + sub_sink->set_name("sub_sink"); + + auto exp_sink = pointwise(sub_sink, exp_attributes); + exp_sink->set_name("exp_sink"); + + auto temp_sum = reduction(exp_output, sum_attributes); + temp_sum->set_name("SumExp_elements").set_dim({b, h, s_q, 1}).set_stride({h * s_q, s_q, 1, 1}); + + auto add_attributes = Pointwise_attributes().set_name("add_sink").set_mode(PointwiseMode_t::ADD); + pointwise(temp_sum, exp_sink, add_attributes, sum_output); + } else { + reduction(exp_output, sum_attributes, sum_output); + } + + // Set to virtual when: + // - softmax stats in not requested + // - max and sum_exp are not requested + if (!has_stats() && !has_max() && !has_sum_exp()) { + auto softmax_stats = std::make_shared(); + softmax_stats->set_is_virtual(true); + attributes.outputs[Softmax_attributes::output_names::Stats] = softmax_stats; + } + + if (has_stats()) { + auto log_attributes = Pointwise_attributes().set_name("log").set_mode(PointwiseMode_t::LOG); + auto const& log_output = pointwise(sum_output, log_attributes); + log_output->set_dim({b, h, s_q, 1}).set_stride({h * s_q, s_q, 1, 1}); + + auto add_attributes = Pointwise_attributes().set_name("add").set_mode(PointwiseMode_t::ADD); + // Special non-functional-style call. Needed because output already created and provided to user. + pointwise( + max_output, log_output, add_attributes, attributes.outputs[Softmax_attributes::output_names::Stats]); + } + + auto div_attributes = Pointwise_attributes().set_name("div").set_mode(PointwiseMode_t::DIV); + // Special non-functional-style call. Needed because output already created and provided to user. + pointwise(exp_output, sum_output, div_attributes, attributes.outputs[Softmax_attributes::output_names::S]); + + return {error_code_t::OK, ""}; + } +}; + +// Newer implementation of softmax node that represents the operation as a single backend operation. +// This is used for cuDNN versions v9.21.0 and later. +class UnifiedSoftmaxNode : public SoftmaxNodeBase { + public: + UnifiedSoftmaxNode(Softmax_attributes&& attributes_, detail::Context const& context) + : SoftmaxNodeBase(std::move(attributes_), context) {} + + Type + getType() override final { + return Type::SOFTMAX; + } + + error_t + pre_validate_node() const override final { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Validating UnifiedSoftmaxNode " << attributes.name); + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set& uids_involved_in_operations, + std::vector>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map>& tensors) const override final { + getLogger() << "[cudnn_frontend] INFO: " << "Building UnifiedSoftmaxNode operations " << attributes.name + << std::endl; + auto cudnn_ver_error = error_t{error_code_t::GRAPH_NOT_SUPPORTED, "UnifiedSoftmaxNode requires cuDNN v9.21.0"}; + +#if (CUDNN_VERSION >= 92100) + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(92100, cudnn_ver_error); + CUDNN_FRONTEND_UNUSED(operations); + auto softmax_operation = + make_shared_backend_pointer((cudnnBackendDescriptorType_t)CUDNN_BACKEND_OPERATION_SOFTMAX_DESCRIPTOR); + + // Set input tensor P + auto P = attributes.inputs.find(Softmax_attributes::input_names::P)->second; + auto backend_p = tensors[P->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(softmax_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SOFTMAX_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_p)); + + // Set output tensor S + auto S = attributes.outputs.find(Softmax_attributes::output_names::S)->second; + auto backend_s = tensors[S->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(softmax_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SOFTMAX_YDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_s)); + + // Set optional SINK tensor if present + if (has_sink()) { + auto SINK = attributes.inputs.find(Softmax_attributes::input_names::SINK)->second; + auto backend_sink = tensors[SINK->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(softmax_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SOFTMAX_SINK_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_sink)); + } + + // Set optional Stats output tensor if present + if (has_stats()) { + auto Stats = attributes.outputs.find(Softmax_attributes::output_names::Stats)->second; + auto backend_stats = tensors[Stats->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(softmax_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SOFTMAX_STATS_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_stats)); + } + + // Set optional Max output tensor if present + if (has_max()) { + auto Max = attributes.outputs.find(Softmax_attributes::output_names::Max)->second; + auto backend_max = tensors[Max->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(softmax_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SOFTMAX_MAX_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_max)); + } + + // Set optional Sum_exp output tensor if present + if (has_sum_exp()) { + auto Sum_exp = attributes.outputs.find(Softmax_attributes::output_names::Sum_exp)->second; + auto backend_sum_exp = tensors[Sum_exp->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(softmax_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_SOFTMAX_SUM_EXP_DESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_sum_exp)); + } + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(softmax_operation->get_backend_descriptor())); + + raw_operations.push_back(softmax_operation); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); + return {error_code_t::OK, ""}; +#else + CUDNN_FRONTEND_UNUSED(uids_involved_in_operations); + CUDNN_FRONTEND_UNUSED(operations); + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FRONTEND_UNUSED(tensors); + return cudnn_ver_error; +#endif // CUDNN_VERSION >= 92100 + } +}; + +// Factory function implementation for INode::softmax +// This function will create the appropriate node based on the cuDNN version. +inline void +INode::softmax(std::shared_ptr p, + Softmax_attributes attributes, + std::shared_ptr s, + std::shared_ptr stats, + std::shared_ptr max, + std::shared_ptr sum_exp) { + attributes.inputs[Softmax_attributes::input_names::P] = p; + attributes.outputs[Softmax_attributes::output_names::S] = s; + if (stats) { + attributes.outputs[Softmax_attributes::output_names::Stats] = stats; + } + if (max) { + attributes.outputs[Softmax_attributes::output_names::Max] = max; + } + if (sum_exp) { + attributes.outputs[Softmax_attributes::output_names::Sum_exp] = sum_exp; + } + + // Newer versions of cuDNN can represent the softmax as a single operation. + if (std::min(detail::get_compiled_version(), detail::get_backend_version()) >= 92100) { + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + } else { + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + } +} + +} // namespace cudnn_frontend::graph diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node/transpose.h b/deps/cudnn-frontend/include/cudnn_frontend/node/transpose.h new file mode 100644 index 0000000..7f764be --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node/transpose.h @@ -0,0 +1,164 @@ +#pragma once + +#include "../../cudnn_frontend_Logging.h" + +#include "../graph_helpers.h" +#include "../node_interface.h" + +namespace cudnn_frontend::graph { + +class TransposeNode : public NodeCRTP { + public: + Transpose_attributes attributes; + + TransposeNode(Transpose_attributes&& attributes_, detail::Context const& context) + : NodeCRTP(context), attributes(std::move(attributes_)) {} + + Type + getType() override final { + return Type::TRANSPOSE; + } + + error_t + infer_properties_node() override final { + getLogger() << "[cudnn_frontend] INFO: Inferrencing properties for transpose node " << attributes.name + << std::endl; + + attributes.fill_from_context(context); + + auto const& X_tensor = attributes.inputs[Transpose_attributes::input_names::X]; + auto Y_tensor = attributes.outputs[Transpose_attributes::output_names::Y]; + + // Get input properties + auto const& input_dim = X_tensor->get_dim(); + auto const& input_stride = X_tensor->get_stride(); + auto const& input_data_type = X_tensor->get_data_type(); + auto const& permutation = attributes.get_permutation(); + + // Validate permutation + RETURN_CUDNN_FRONTEND_ERROR_IF( + permutation.empty(), error_code_t::ATTRIBUTE_NOT_SET, "Permutation must be set for transpose operation."); + + RETURN_CUDNN_FRONTEND_ERROR_IF(permutation.size() != input_dim.size(), + error_code_t::INVALID_VALUE, + "Permutation size must match input tensor dimensionality."); + + // Check that permutation is a valid permutation (contains each index 0 to n-1 exactly once) + std::vector seen(permutation.size(), false); + for (auto idx : permutation) { + RETURN_CUDNN_FRONTEND_ERROR_IF(idx < 0 || idx >= static_cast(permutation.size()), + error_code_t::INVALID_VALUE, + "Permutation indices must be in range [0, n-1]."); + RETURN_CUDNN_FRONTEND_ERROR_IF( + seen[idx], error_code_t::INVALID_VALUE, "Permutation indices must be unique."); + seen[idx] = true; + } + + // Infer output dimensions by permuting input dimensions + std::vector output_dim(input_dim.size()); + for (size_t i = 0; i < permutation.size(); ++i) { + output_dim[i] = input_dim[permutation[i]]; + } + + // Infer output strides by permuting input strides + std::vector output_stride(input_stride.size()); + for (size_t i = 0; i < permutation.size(); ++i) { + output_stride[i] = input_stride[permutation[i]]; + } + + // Set output tensor properties + if (Y_tensor->get_dim().empty()) { + Y_tensor->set_dim(output_dim); + } + if (Y_tensor->get_stride().empty()) { + Y_tensor->set_stride(output_stride); + } + if (Y_tensor->get_data_type() == DataType_t::NOT_SET) { + Y_tensor->set_data_type(input_data_type); + } + + RETURN_CUDNN_FRONTEND_ERROR_IF(Y_tensor->get_data_type() != input_data_type, + error_code_t::INVALID_VALUE, + "Output and input tensor data types must match for transpose operation."); + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_operations( + std::unordered_set& uids_involved_in_operations, + std::vector>& operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map>& tensors) const override final { + getLogger() << "[cudnn_frontend] INFO: " << "Building TransposeNode operations " << attributes.name + << std::endl; + +#if (CUDNN_VERSION >= 92200) + // cuDNN >= 9.22.0: Use native backend transpose operation + auto cudnn_ver_error = error_t{error_code_t::GRAPH_NOT_SUPPORTED, "Transpose requires cuDNN v9.22.0"}; + NV_CUDNN_FE_DYNAMIC_CHECK_CUDNN_BACKEND_VERSION(92200, cudnn_ver_error); + CUDNN_FRONTEND_UNUSED(operations); + + auto transpose_operation = make_shared_backend_pointer(CUDNN_BACKEND_OPERATION_TRANSPOSE_DESCRIPTOR); + + // Set input tensor + auto X = attributes.inputs.at(Transpose_attributes::input_names::X); + auto backend_x = tensors[X->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(transpose_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_TRANSPOSE_XDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_x)); + + // Set output tensor + auto Y = attributes.outputs.at(Transpose_attributes::output_names::Y); + auto backend_y = tensors[Y->get_uid()]->get_desc()->get_backend_descriptor(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(transpose_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_TRANSPOSE_YDESC, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &backend_y)); + + // Set permutation + auto permutation = attributes.get_permutation(); + _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(transpose_operation->get_backend_descriptor(), + CUDNN_ATTR_OPERATION_TRANSPOSE_PERMUTATION, + CUDNN_TYPE_INT64, + permutation.size(), + permutation.data())); + + _CUDNN_CHECK_CUDNN_ERROR(detail::finalize(transpose_operation->get_backend_descriptor())); + + raw_operations.push_back(transpose_operation); + + auto const& non_virtual_uids = attributes.get_non_virtual_uids(); + uids_involved_in_operations.insert(non_virtual_uids.begin(), non_virtual_uids.end()); +#else + CUDNN_FRONTEND_UNUSED(operations); + CUDNN_FRONTEND_UNUSED(raw_operations); + CUDNN_FRONTEND_UNUSED(tensors); + CUDNN_FRONTEND_UNUSED(uids_involved_in_operations); + return {error_code_t::GRAPH_NOT_SUPPORTED, "Transpose operation requires cuDNN version >= 9.22.0"}; +#endif + return {error_code_t::OK, ""}; + } + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const override final { + j = attributes; + j.update(R"( {"tag": "TRANSPOSE"})"_json); + } +#endif +}; + +inline std::shared_ptr +INode::transpose(std::shared_ptr input, Transpose_attributes attributes) { + attributes.inputs[Transpose_attributes::input_names::X] = input; + auto Y = attributes.outputs[Transpose_attributes::output_names::Y] = output_tensor(attributes.name + "::Y"); + + sub_nodes.emplace_back(std::make_unique(std::move(attributes), context)); + return Y; +} + +} // namespace cudnn_frontend::graph diff --git a/deps/cudnn-frontend/include/cudnn_frontend/node_interface.h b/deps/cudnn-frontend/include/cudnn_frontend/node_interface.h new file mode 100644 index 0000000..8d46ece --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/node_interface.h @@ -0,0 +1,557 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include +#include + +#include "../cudnn_frontend_Tensor.h" +#include "../cudnn_frontend_Operation.h" +#include "../cudnn_frontend_OperationGraph.h" +#include "../cudnn_frontend_ExecutionPlan.h" +#include "../cudnn_frontend_VariantPack.h" +#include "../cudnn_frontend_shim.h" + +#include "cudnn_interface.h" + +#include "graph_properties.h" + +namespace cudnn_frontend { + +namespace graph { + +class BatchNormNode; +class DBNNode; +class ConcatenateNode; +class MatmulNode; +class MatmulFP8Node; +class PointwiseNode; +class ReductionNode; +class ResampleNode; +class ReshapeNode; +class RngNode; +class CompositeSoftmaxNode; +class UnifiedSoftmaxNode; +class MoeGroupedMatmulNode; +class UnifiedDiagonalBandMaskNode; +class TransposeNode; +class SliceNode; + +// Interface for all nodes to follow. +class INode { + public: + // A closed set of types that are allowed to be passed by value today + using pass_by_values_t = Tensor_attributes::pass_by_values_t; + + detail::Context context; + + protected: + // Will eventually be moved to Graph class + std::unordered_set> full_graph_outputs; + std::shared_ptr + output_tensor(std::string const& name) { + auto tensor = std::make_shared(); + tensor->set_name(name).set_is_virtual(true); + full_graph_outputs.insert(tensor); + return tensor; + } + + private: + virtual error_t + pre_validate_node() const { + return {error_code_t::OK, ""}; + }; + + virtual error_t + infer_properties_node() = 0; + + virtual error_t + expand_node() { + return {error_code_t::OK, ""}; + }; + + virtual error_t + post_validate_node() const { + return {error_code_t::OK, ""}; + }; + + virtual int64_t + get_fe_workspace_size_node() const { + return 0; + } + + virtual error_t + collect_pass_by_value_tensors_node(std::unordered_map&) const { + return {error_code_t::OK, ""}; + }; + + virtual error_t + collect_variant_pack_replacements_node( + std::unordered_map>&) const { + return {error_code_t::OK, ""}; + }; + + virtual error_t + collect_tensors_to_dump_node(std::vector, char>>&) const { + return {error_code_t::OK, ""}; + }; + + virtual error_t + create_cudnn_tensors_node( + std::unordered_map>& uid_to_backend_tensors, + int64_t& potential_uid, + std::unordered_set const& used_uids) const = 0; + + virtual error_t + collect_tensors_in_workspace_node( + std::unordered_map>>&, + int64_t&) const { + return {error_code_t::OK, ""}; + } + + protected: + // Type of each node. Nodes can either be a composite (value COMPOSITE) or + // one of the other primitive types. Primitives types are nothing but + // cudnn operations. + enum class Type { + COMPOSITE, + BATCHNORM, + BATCHNORM_INFERENCE, + BN_FINALIZE, + CONVOLUTION, + DBN, + DBN_WEIGHT, + DLN, + DIN, + DGRAD, + DRMSNorm, + GENSTATS, + LAYERNORM, + INSTANCENORM, + MATMUL, + POINTWISE, + REDUCTION, + RESAMPLE, + RESHAPE, + RMSNORM, + RNG, + SLICE, + TRANSPOSE, + WGRAD, + PAGED_CACHE_LOAD, + BLOCK_SCALE_QUANTIZE, + BLOCK_SCALE_DEQUANTIZE, + CONCATENATE, + ADALAYERNORM, + DADALAYERNORM, + SDPA, + ROPE, + ROPE_BWD, + MOE_GROUPED_MATMUL, + MOE_GROUPED_MATMUL_BWD, + DIAGONAL_BAND_MASK, + SOFTMAX, + }; + Type tag; + + inline void + matmul(std::shared_ptr a, + std::shared_ptr b, + Matmul_attributes attributes, + std::shared_ptr c); + + void + matmul_fp8(std::shared_ptr a, + std::shared_ptr b, + std::shared_ptr descale_a, + std::shared_ptr descale_b, + std::shared_ptr scale_c, + Matmul_fp8_attributes attributes, + std::shared_ptr c, + std::shared_ptr amax_c); + + void + softmax(std::shared_ptr p, + Softmax_attributes attributes, + std::shared_ptr s, + std::shared_ptr stats, + std::shared_ptr max, + std::shared_ptr sum_exp); + + void + softmax(std::shared_ptr p, + Softmax_attributes attributes, + std::shared_ptr s, + std::shared_ptr m, + std::shared_ptr zinv); + + void + pointwise(std::shared_ptr a, + Pointwise_attributes attributes, + std::shared_ptr c); + + void + pointwise(std::shared_ptr a, + std::shared_ptr b, + Pointwise_attributes attributes, + std::shared_ptr c); + + void + pointwise(std::shared_ptr a, + std::shared_ptr b, + std::shared_ptr c, + Pointwise_attributes attributes, + std::shared_ptr d); + + void + reduction(std::shared_ptr a, + Reduction_attributes attributes, + std::shared_ptr c); + + void + reduction(std::shared_ptr a, + std::shared_ptr group_offset, + Reduction_attributes attributes, + std::shared_ptr c); + + void + rng(std::shared_ptr seed, + std::shared_ptr offset, + Rng_attributes attributes, + std::shared_ptr y); + + void + paged_cache_load(std::shared_ptr container, + std::shared_ptr seqLen, + std::shared_ptr pageTable, + PagedCacheLoad_attributes attributes, + std::shared_ptr yOut); + + void + block_scale_quantize(std::shared_ptr x, + Block_scale_quantize_attributes attributes, + std::shared_ptr y, + std::shared_ptr scale); + + void + block_scale_dequantize(std::shared_ptr x, + std::shared_ptr scale, + Block_scale_dequantize_attributes attributes, + std::shared_ptr y); + + void + concatenate(std::vector> x, + Concatenate_attributes attributes, + std::shared_ptr y); + + void + moe_grouped_matmul(std::shared_ptr token, + std::shared_ptr weight, + std::shared_ptr first_token_offset, + std::shared_ptr token_index, + std::shared_ptr token_ks, + Moe_grouped_matmul_attributes attributes, + std::shared_ptr output); + + void + moe_grouped_matmul_bwd(std::shared_ptr doutput, + std::shared_ptr token, + std::shared_ptr first_token_offset, + Moe_grouped_matmul_bwd_attributes attributes, + std::shared_ptr dweight); + + void + transpose(std::shared_ptr input, + Transpose_attributes attributes, + std::shared_ptr output); + + void + slice(std::shared_ptr input, + Slice_attributes attributes, + std::shared_ptr output); + error_t + validate_subtree() { + // pre validate to catch errors early + // Otherwise code reability decreases in expand_and_infer + CHECK_CUDNN_FRONTEND_ERROR(pre_validate_node()); + CHECK_CUDNN_FRONTEND_ERROR(infer_properties_node()); + for (auto const& sub_node : sub_nodes) { + CHECK_CUDNN_FRONTEND_ERROR(sub_node->validate_subtree()); + } + CHECK_CUDNN_FRONTEND_ERROR(post_validate_node()); + return {error_code_t::OK, ""}; + } + + error_t + expand_subtree() { + // Validate self + CHECK_CUDNN_FRONTEND_ERROR(pre_validate_node()); + CHECK_CUDNN_FRONTEND_ERROR(infer_properties_node()); + CHECK_CUDNN_FRONTEND_ERROR(expand_node()); + for (auto const& sub_node : sub_nodes) { + CHECK_CUDNN_FRONTEND_ERROR(sub_node->expand_subtree()); + } + CHECK_CUDNN_FRONTEND_ERROR(post_validate_node()); + return {error_code_t::OK, ""}; + } + + // Creates cudnn tensors for each node (and its sub nodes) + error_t + create_cudnn_tensors_subtree( + std::unordered_map>& uid_to_backend_tensors, + int64_t& potential_uid, + std::unordered_set const& used_uids) const { + CHECK_CUDNN_FRONTEND_ERROR(create_cudnn_tensors_node(uid_to_backend_tensors, potential_uid, used_uids)); + for (auto const& sub_node : sub_nodes) { + CHECK_CUDNN_FRONTEND_ERROR( + sub_node->create_cudnn_tensors_subtree(uid_to_backend_tensors, potential_uid, used_uids)); + } + return {error_code_t::OK, ""}; + } + + error_t + collect_pass_by_value_tensors_subtree( + std::unordered_map& tensor_to_pass_by_value) const { + CHECK_CUDNN_FRONTEND_ERROR(collect_pass_by_value_tensors_node(tensor_to_pass_by_value)); + for (auto const& sub_node : sub_nodes) { + CHECK_CUDNN_FRONTEND_ERROR(sub_node->collect_pass_by_value_tensors_subtree(tensor_to_pass_by_value)); + } + return {error_code_t::OK, ""}; + } + + error_t + collect_tensors_in_workspace_subtree( + std::unordered_map>>& + worskspace_modifications, + int64_t& offset) const { + CHECK_CUDNN_FRONTEND_ERROR(collect_tensors_in_workspace_node(worskspace_modifications, offset)); + offset = get_fe_workspace_size_node(); + for (auto const& sub_node : sub_nodes) { + CHECK_CUDNN_FRONTEND_ERROR( + sub_node->collect_tensors_in_workspace_subtree(worskspace_modifications, offset)); + offset += sub_node->get_fe_workspace_size_node(); + } + return {error_code_t::OK, ""}; + } + + error_t + collect_variant_pack_replacements_subtree( + std::unordered_map>& replacements) + const { + CHECK_CUDNN_FRONTEND_ERROR(collect_variant_pack_replacements_node(replacements)); + for (auto const& sub_node : sub_nodes) { + CHECK_CUDNN_FRONTEND_ERROR(sub_node->collect_variant_pack_replacements_subtree(replacements)); + } + return {error_code_t::OK, ""}; + } + + error_t + collect_tensors_to_dump_subtree( + std::vector, char>>& tensors_to_dump) const { + CHECK_CUDNN_FRONTEND_ERROR(collect_tensors_to_dump_node(tensors_to_dump)); + for (auto const& sub_node : sub_nodes) { + CHECK_CUDNN_FRONTEND_ERROR(sub_node->collect_tensors_to_dump_subtree(tensors_to_dump)); + } + return {error_code_t::OK, ""}; + } + + int64_t + get_fe_workspace_size_subtree() const { + int64_t fe_workspace_size = get_fe_workspace_size_node(); + for (auto const& sub_node : sub_nodes) { + fe_workspace_size += sub_node->get_fe_workspace_size_subtree(); + } + return fe_workspace_size; + } + + // Creates cudnn operation for each node (and its sub nodes) + // Only INode that map to a primitive cudnn operation need to specialize. + virtual error_t + create_cudnn_operations( + std::unordered_set& uids_involved_in_operation, + std::vector>& backend_operations, + managed_backend_descriptor_t& raw_operations, + std::unordered_map>& uid_to_backend_tensors) const { + for (auto const& sub_node : sub_nodes) { + CHECK_CUDNN_FRONTEND_ERROR(sub_node->create_cudnn_operations( + uids_involved_in_operation, backend_operations, raw_operations, uid_to_backend_tensors)); + } + return {error_code_t::OK, ""}; + } + + // An implicitly topological-sorted vector of sub nodes. + // The sorted order is a side effect of functional API. + std::vector> sub_nodes; + + public: + virtual Type + getType() = 0; + + virtual std::pair> + override_heuristics_query() const { + return {-1, {}}; + } + + std::shared_ptr matmul(std::shared_ptr, + std::shared_ptr, + Matmul_attributes); + + std::shared_ptr pointwise(std::shared_ptr, Pointwise_attributes); + std::shared_ptr pointwise(std::shared_ptr, + std::shared_ptr, + Pointwise_attributes); + std::shared_ptr pointwise(std::shared_ptr, + std::shared_ptr, + std::shared_ptr, + Pointwise_attributes); + + std::shared_ptr reduction(std::shared_ptr, Reduction_attributes); + std::shared_ptr reduction(std::shared_ptr, + std::shared_ptr, + Reduction_attributes); + std::array, 2> resample(std::shared_ptr, Resample_attributes); + std::shared_ptr reshape(std::shared_ptr, Reshape_attributes); + std::shared_ptr transpose(std::shared_ptr, Transpose_attributes); + std::shared_ptr slice(std::shared_ptr, Slice_attributes); + + std::shared_ptr rng(std::shared_ptr, + std::shared_ptr, + Rng_attributes); + + std::shared_ptr + diagonal_band_mask(std::shared_ptr x, + std::shared_ptr b, + std::shared_ptr seq_len_q, + std::shared_ptr seq_len_kv, + std::shared_ptr cu_seq_len_q, + std::shared_ptr cu_seq_len_kv, + std::shared_ptr left_bound, + std::shared_ptr shift_right_bound, + DiagonalBandMask_attributes attributes); + + INode(detail::Context const& context) : context(context) {} + + // Make sure each node implements a public serialize function +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + virtual void + serialize(json& j) const = 0; +#endif + + virtual size_t + key() { +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB + json j; + serialize(j); + return std::hash{}(j); +#else + return 1; +#endif + } + + virtual ~INode() = default; +}; + +#ifndef CUDNN_FRONTEND_SKIP_JSON_LIB +[[maybe_unused]] static void +to_json(json& j, const INode& p) { + p.serialize(j); +} +#endif + +template +class NodeCRTP : public INode { + DerivedT& + self() { + return *static_cast(this); + } + DerivedT const& + self() const { + return *static_cast(this); + } + + error_t + collect_pass_by_value_tensors_node( + std::unordered_map& tensor_to_pass_by_value) const override final { + CHECK_CUDNN_FRONTEND_ERROR(self().attributes.fill_pass_by_value(tensor_to_pass_by_value)); + + return {error_code_t::OK, ""}; + } + + error_t + create_cudnn_tensors_node(std::unordered_map>& tensors, + int64_t& potential_uid, + std::unordered_set const& used_uids) const override { + CUDNN_FE_LOG_LABEL_ENDL("INFO: Creating cudnn tensors for node named '" << self().attributes.name << "':"); + + if constexpr (std::is_same_v) { + for (auto const& tensor : self().attributes.inputs) { + if (tensor) { + CHECK_CUDNN_FRONTEND_ERROR(detail::create_cudnn_tensor(tensor, tensors, potential_uid, used_uids)); + } + } + } else { + for (auto const& [name, tensor] : self().attributes.inputs) { + (void)name; + if (tensor) { + CHECK_CUDNN_FRONTEND_ERROR(detail::create_cudnn_tensor(tensor, tensors, potential_uid, used_uids)); + } + } + } + + for (auto const& [name, tensor] : self().attributes.outputs) { + (void)name; + if (tensor) { + CHECK_CUDNN_FRONTEND_ERROR(detail::create_cudnn_tensor(tensor, tensors, potential_uid, used_uids)); + } + } + + // Handle special case of BN where peer_stats is also an input + if constexpr (std::is_same_v || std::is_same_v) { + // Special case in BN where peer stats is also an input but is not present in inputs map + for (auto const& tensor : self().attributes.peer_stats) { + if (tensor) { + CHECK_CUDNN_FRONTEND_ERROR(detail::create_cudnn_tensor(tensor, tensors, potential_uid, used_uids)); + } + } + } + + return {error_code_t::OK, ""}; + } + + protected: + using INode::INode; +}; + +#define CUDNN_FE_VALIDATE_TENSOR_(port, map_) \ + { \ + auto t = map_.find(port); \ + bool const has_t = (t != map_.end()) && (t->second != nullptr); \ + RETURN_CUDNN_FRONTEND_ERROR_IF( \ + !has_t, error_code_t::ATTRIBUTE_NOT_SET, std::string("Tensor ") + #port + " not set"); \ + } + +#define CUDNN_FE_VALIDATE_AND_ASSIGN_TENSOR_(tensor, port, map_) \ + auto tensor = map_.find(port); \ + { \ + bool const has_t = (tensor != map_.end()) && (tensor->second != nullptr); \ + RETURN_CUDNN_FRONTEND_ERROR_IF( \ + !has_t, error_code_t::ATTRIBUTE_NOT_SET, std::string("Tensor ") + #port + " not set"); \ + } + +#define CUDNN_FE_VALIDATE_INPUT_TENSOR(port) CUDNN_FE_VALIDATE_TENSOR_(port, attributes.inputs) + +#define CUDNN_FE_VALIDATE_OUTPUT_TENSOR(port) CUDNN_FE_VALIDATE_TENSOR_(port, attributes.outputs) + +#define CUDNN_FE_VALIDATE_AND_ASSIGN_INPUT_TENSOR(tensor, port) \ + CUDNN_FE_VALIDATE_AND_ASSIGN_TENSOR_(tensor, port, attributes.inputs) + +#define CUDNN_FE_VALIDATE_AND_ASSIGN_OUTPUT_TENSOR(tensor, port) \ + CUDNN_FE_VALIDATE_AND_ASSIGN_TENSOR_(tensor, port, attributes.outputs) + +} // namespace graph + +} // namespace cudnn_frontend diff --git a/deps/cudnn-frontend/include/cudnn_frontend/plans.h b/deps/cudnn-frontend/include/cudnn_frontend/plans.h new file mode 100644 index 0000000..de51929 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/plans.h @@ -0,0 +1,1175 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "knobs.h" +#include "../cudnn_frontend_EngineConfig.h" +#include "../cudnn_frontend_Logging.h" +#include "graph_helpers.h" + +#include "backend/execution_helpers.h" +#include "backend/plan_helpers.h" +#include "experimental/sm90_sdpa_prefill_engine.h" +#include "experimental/sm100_sdpa_prefill_engine.h" +#include "experimental/sm100_rms_norm_silu_engine.h" + +namespace cudnn_frontend { + +namespace detail { + +inline error_t +execute(cudnnHandle_t handle, + ExecutionPlan* plan, + std::vector& device_ptrs, + std::vector const& uids, + void* workspace_ptr, + std::vector const& override_uids, + std::vector> const& override_shapes, + std::vector> const& override_strides) { + if (plan == nullptr) { + return {error_code_t::GRAPH_EXECUTION_FAILED, "No plan found to execute!"}; + } + CUDNN_FE_LOG_LABEL_ENDL("INFO: Executing " << plan->getTag() << "..."); + + backend_descriptor variant_pack_descriptor(CUDNN_BACKEND_VARIANT_PACK_DESCRIPTOR); + RETURN_CUDNN_FRONTEND_ERROR_IF(variant_pack_descriptor.get_status() != CUDNN_STATUS_SUCCESS, + error_code_t::CUDNN_BACKEND_API_FAILED, + "Failed to create variant pack's backend descriptor."); + + CHECK_CUDNN_FRONTEND_ERROR(create_variant_pack( + variant_pack_descriptor, device_ptrs, uids, workspace_ptr, override_uids, override_shapes, override_strides)); + _CUDNN_CHECK_CUDNN_ERROR(execute(handle, plan->get_raw_desc(), variant_pack_descriptor.get_ptr())); + + CUDNN_FE_LOG_LABEL_ENDL("INFO: Executed " << plan->getTag() << "."); + + return {error_code_t::OK, ""}; +} + +inline error_t +execute(cudnnHandle_t handle, + ExecutionPlan* plan, + std::vector& device_ptrs, + std::vector const& uids, + void* workspace_ptr) { + if (plan == nullptr) { + return {error_code_t::GRAPH_EXECUTION_FAILED, "No plan found to execute!"}; + } + CUDNN_FE_LOG_LABEL_ENDL("INFO: Executing " << plan->getTag() << "..."); + + backend_descriptor variant_pack_descriptor(CUDNN_BACKEND_VARIANT_PACK_DESCRIPTOR); + RETURN_CUDNN_FRONTEND_ERROR_IF(variant_pack_descriptor.get_status() != CUDNN_STATUS_SUCCESS, + error_code_t::CUDNN_BACKEND_API_FAILED, + "Failed to create variant pack's backend descriptor."); + + CHECK_CUDNN_FRONTEND_ERROR(create_variant_pack(variant_pack_descriptor, device_ptrs, uids, workspace_ptr)); + _CUDNN_CHECK_CUDNN_ERROR(execute(handle, plan->get_raw_desc(), variant_pack_descriptor.get_ptr())); + + CUDNN_FE_LOG_LABEL_ENDL("INFO: Executed " << plan->getTag() << "."); + + return {error_code_t::OK, ""}; +} + +// Raw-pointer overloads. Array length is inferred from uids.size(). + +inline error_t +execute(cudnnHandle_t handle, + ExecutionPlan* plan, + void* const* device_ptrs, + std::vector const& uids, + void* workspace_ptr) { + if (plan == nullptr) { + return {error_code_t::GRAPH_EXECUTION_FAILED, "No plan found to execute!"}; + } + CUDNN_FE_LOG_LABEL_ENDL("INFO: Executing " << plan->getTag() << "..."); + + backend_descriptor variant_pack_descriptor(CUDNN_BACKEND_VARIANT_PACK_DESCRIPTOR); + RETURN_CUDNN_FRONTEND_ERROR_IF(variant_pack_descriptor.get_status() != CUDNN_STATUS_SUCCESS, + error_code_t::CUDNN_BACKEND_API_FAILED, + "Failed to create variant pack's backend descriptor."); + + CHECK_CUDNN_FRONTEND_ERROR(create_variant_pack(variant_pack_descriptor, device_ptrs, uids, workspace_ptr)); + _CUDNN_CHECK_CUDNN_ERROR(execute(handle, plan->get_raw_desc(), variant_pack_descriptor.get_ptr())); + + CUDNN_FE_LOG_LABEL_ENDL("INFO: Executed " << plan->getTag() << "."); + return {error_code_t::OK, ""}; +} + +inline error_t +execute(cudnnHandle_t handle, + ExecutionPlan* plan, + void* const* device_ptrs, + std::vector const& uids, + void* workspace_ptr, + std::vector const& override_uids, + std::vector> const& override_shapes, + std::vector> const& override_strides) { + if (plan == nullptr) { + return {error_code_t::GRAPH_EXECUTION_FAILED, "No plan found to execute!"}; + } + CUDNN_FE_LOG_LABEL_ENDL("INFO: Executing " << plan->getTag() << "..."); + + backend_descriptor variant_pack_descriptor(CUDNN_BACKEND_VARIANT_PACK_DESCRIPTOR); + RETURN_CUDNN_FRONTEND_ERROR_IF(variant_pack_descriptor.get_status() != CUDNN_STATUS_SUCCESS, + error_code_t::CUDNN_BACKEND_API_FAILED, + "Failed to create variant pack's backend descriptor."); + + CHECK_CUDNN_FRONTEND_ERROR(create_variant_pack( + variant_pack_descriptor, device_ptrs, uids, workspace_ptr, override_uids, override_shapes, override_strides)); + _CUDNN_CHECK_CUDNN_ERROR(execute(handle, plan->get_raw_desc(), variant_pack_descriptor.get_ptr())); + + CUDNN_FE_LOG_LABEL_ENDL("INFO: Executed " << plan->getTag() << "."); + return {error_code_t::OK, ""}; +} + +inline error_t +query_cudnn_heuristics_impl(std::shared_ptr const& operation_graph, + cudnn_frontend::EngineConfigList& configs, + std::vector const& modes, + int32_t sm_count, + std::shared_ptr device_properties = nullptr) { + RETURN_CUDNN_FRONTEND_ERROR_IF( + operation_graph == nullptr, + error_code_t::HEURISTIC_QUERY_FAILED, + "Empty operation graph provided. Did you forget to call graph.build_operation_graph()?"); + + auto const& operation_graph_tag = operation_graph->getTag(); + CUDNN_FE_LOG_LABEL_ENDL("INFO: " << " Getting plan from heuristics for " << operation_graph_tag << " ..."); + + std::vector statuses; +#ifdef NV_CUDNN_DISABLE_EXCEPTION + // disable exception macro is defined. Calling build will not throw. + // Check status of desc and return error. + statuses = cudnn_frontend::get_heuristics_list( + modes, *operation_graph, allowAllConfig, configs, true, sm_count, device_properties); +#else + // build() can throw + // wrap in try catch + try { + statuses = cudnn_frontend::get_heuristics_list( + modes, *operation_graph, allowAllConfig, configs, true, sm_count, device_properties); + } catch (cudnn_frontend::cudnnException& e) { + // Silly MSVC error that thinks below condition is constexpr + // RETURN_CUDNN_FRONTEND_ERROR_IF( + // e.getCudnnStatus() != CUDNN_STATUS_SUCCESS, error_code_t::HEURISTIC_QUERY_FAILED, e.what()); + CUDNN_FE_LOG_LABEL("ERROR: " << e.what() << ". "); + CUDNN_FE_LOG(error_code_t::HEURISTIC_QUERY_FAILED << " because querying heuristics failed at " << __FILE__ + << ":" << __LINE__ << "\n"); + return {error_code_t::HEURISTIC_QUERY_FAILED, e.what()}; + } +#endif + + CUDNN_FE_LOG_LABEL("INFO: get_heuristics_list statuses: "); + for (size_t i = 0; i < statuses.size(); i++) { + CUDNN_FE_LOG(cudnn_frontend::to_string(statuses[i]) << " "); + } + CUDNN_FE_LOG(std::endl); + + CUDNN_FE_LOG_LABEL_ENDL("INFO: config list has " << configs.size() << " configurations."); + + if (configs.empty()) { + std::string err_msg = detail::get_last_error_string_(); + CUDNN_FE_LOG_LABEL_ENDL("ERROR: No valid engine configs returned from heuristics.\n" << err_msg); + return {error_code_t::HEURISTIC_QUERY_FAILED, + "No valid engine configs for " + operation_graph_tag + "\n" + err_msg}; + } + return {error_code_t::OK, ""}; +} + +inline error_t +create_cudnn_execution_plan(std::shared_ptr& plan, + std::string const& serialized_data, + cudnnHandle_t handle) { + auto&& plan_builder = cudnn_frontend::ExecutionPlanBuilder(); + + plan_builder.setHandle(handle); + +#ifdef NV_CUDNN_DISABLE_EXCEPTION + // disable exception macro is defined. Calling build will not throw. + // Check status of desc and return error. + auto built_plan = plan_builder.loadFromJson(serialized_data); + RETURN_CUDNN_FRONTEND_ERROR_IF(built_plan.get_status() != CUDNN_STATUS_SUCCESS, + error_code_t::GRAPH_EXECUTION_PLAN_CREATION_FAILED, + built_plan.get_error()); + plan = std::make_shared(std::move(built_plan)); +#else + // build() can throw + // wrap in try catch + try { + auto built_plan = plan_builder.loadFromJson(serialized_data); + plan = std::make_shared(std::move(built_plan)); + } catch (cudnn_frontend::cudnnException& e) { + // Silly MSVC error that thinks below condition is constexpr + // RETURN_CUDNN_FRONTEND_ERROR_IF( + // e.getCudnnStatus() != CUDNN_STATUS_SUCCESS, error_code_t::GRAPH_EXECUTION_PLAN_CREATION_FAILED, + // e.what()); + CUDNN_FE_LOG_LABEL(" ERROR: " << e.what() << ". "); + CUDNN_FE_LOG(error_code_t::GRAPH_EXECUTION_PLAN_CREATION_FAILED << " because plan building failed at " + << __FILE__ << ":" << __LINE__ << "\n"); + return {error_code_t::GRAPH_EXECUTION_PLAN_CREATION_FAILED, e.what()}; + } +#endif + + return {error_code_t::OK, ""}; +} + +inline error_t +create_cudnn_execution_plan(std::shared_ptr& plan, + ManagedOpaqueDescriptor const& config, + std::string const& operation_graph_tag, + std::shared_ptr kernel_cache) { + auto&& plan_builder = cudnn_frontend::ExecutionPlanBuilder(); + + plan_builder.setEngineConfig(config, operation_graph_tag).setKernelCache(kernel_cache); + +#ifdef NV_CUDNN_DISABLE_EXCEPTION + // disable exception macro is defined. Calling build will not throw. + // Check status of desc and return error. + auto built_plan = plan_builder.build(); + RETURN_CUDNN_FRONTEND_ERROR_IF(built_plan.get_status() != CUDNN_STATUS_SUCCESS, + error_code_t::GRAPH_EXECUTION_PLAN_CREATION_FAILED, + built_plan.get_error()); + plan = std::make_shared(std::move(built_plan)); +#else + // build() can throw + // wrap in try catch + try { + auto built_plan = plan_builder.build(); + plan = std::make_shared(std::move(built_plan)); + } catch (cudnn_frontend::cudnnException& e) { + // Silly MSVC error that thinks below condition is constexpr + // RETURN_CUDNN_FRONTEND_ERROR_IF( + // e.getCudnnStatus() != CUDNN_STATUS_SUCCESS, error_code_t::GRAPH_EXECUTION_PLAN_CREATION_FAILED, + // e.what()); + CUDNN_FE_LOG_LABEL("ERROR: " << e.what() << ". "); + CUDNN_FE_LOG(error_code_t::GRAPH_EXECUTION_PLAN_CREATION_FAILED << " because plan building failed at " + << __FILE__ << ":" << __LINE__ << "\n"); + return {error_code_t::GRAPH_EXECUTION_PLAN_CREATION_FAILED, e.what()}; + } +#endif + + return {error_code_t::OK, ""}; +} + +} // namespace detail + +namespace graph { +class Execution_plan_list { + std::string operation_tag; + + std::vector barred_indices; + std::shared_ptr kernel_cache = nullptr; + + int64_t max_workspace_allowed = std::numeric_limits::max(); + int64_t max_shared_mem_allowed = 1024 * 1024 * 1024; // Crazy high number (2GB) which will never be hit + + std::vector barred_engine_names = {}; + EngineConfigList engine_configs; + + error_t + _build_plan_at_index_impl(int64_t index) { + if (execution_plans[index] == nullptr) { + CHECK_CUDNN_FRONTEND_ERROR(detail::create_cudnn_execution_plan( + execution_plans[index], engine_configs[index], operation_tag, kernel_cache)); + } + + auto is_blocked = [](std::string const& full_name, std::vector const& blocked_names) -> bool { + for (auto const& blocked_name : blocked_names) { + if (full_name.find(blocked_name) != std::string::npos) { + return true; + } + } + return false; + }; + auto const& plan_tag = execution_plans[index]->getTag(); + if (is_blocked(plan_tag, barred_engine_names)) { + barred_indices[index] = true; + + return {error_code_t::GRAPH_EXECUTION_PLAN_CREATION_FAILED, + "[cudnn_frontend] Error: Deselecting execution plan with name " + plan_tag + " at position " + + std::to_string(index)}; + } + + // workspace check for 9.2+ is already done at engine config level + if (detail::get_backend_version() < 90200 || detail::get_compiled_version() < 90200) { + if (execution_plans[index]->getWorkspaceSize() > max_workspace_allowed) { + barred_indices[index] = true; + return {error_code_t::GRAPH_EXECUTION_PLAN_CREATION_FAILED, + "[cudnn_frontend] Error: Workspace size is too large."}; + } + } + + // Sets candidate in case user does not call execute with plan_index later. + candidate = index; + + return {error_code_t::OK, ""}; + } + + public: + std::vector> numeric_notes; + std::vector> behavior_notes; + + std::vector> + execution_plans; // a built plan corresponding to each engine config, irrespective of whether config is + // selected or deselected. + + // Stores position of best plan in above vector of execution plan + int64_t candidate = -1; + + void + set_tag(std::string const& tag) { + operation_tag = tag; + } + void + enqueue_engine_configs(EngineConfigList list) { + std::move(list.begin(), list.end(), back_inserter(engine_configs)); + } + void + set_kernel_cache(std::shared_ptr kernel_cache_) { + kernel_cache = kernel_cache_; + } + + std::vector>& + get_execution_plans() { + return execution_plans; + } + + error_t + query_properties() { + numeric_notes.reserve(engine_configs.size()); + behavior_notes.reserve(engine_configs.size()); + + barred_indices.resize(engine_configs.size(), 0); + execution_plans.resize(engine_configs.size()); + + for (auto& engine_config : engine_configs) { + int64_t elem_count = 0; + std::vector numeric; + std::vector behavior; + + ManagedOpaqueDescriptor extractedEngine = make_shared_backend_pointer(CUDNN_BACKEND_ENGINE_DESCRIPTOR); + cudnnBackendDescriptor_t extractedEngine_ = extractedEngine->get_backend_descriptor(); + auto status = detail::get_attribute(engine_config->get_backend_descriptor(), + CUDNN_ATTR_ENGINECFG_ENGINE, + CUDNN_TYPE_BACKEND_DESCRIPTOR, + 1, + &elem_count, + &extractedEngine_); + RETURN_CUDNN_FRONTEND_ERROR_IF((status != CUDNN_STATUS_SUCCESS), + error_code_t::HEURISTIC_QUERY_FAILED, + "Heuristic query Engine failed."); + + status = detail::get_attribute(extractedEngine_, + CUDNN_ATTR_ENGINE_NUMERICAL_NOTE, + CUDNN_TYPE_NUMERICAL_NOTE, + CUDNN_NUMERICAL_NOTE_TYPE_COUNT, + &elem_count, + nullptr); + RETURN_CUDNN_FRONTEND_ERROR_IF((status != CUDNN_STATUS_SUCCESS), + error_code_t::HEURISTIC_QUERY_FAILED, + "Heuristic query Numerical Note failed"); + + numeric.resize(static_cast(elem_count)); + status = detail::get_attribute(extractedEngine_, + CUDNN_ATTR_ENGINE_NUMERICAL_NOTE, + CUDNN_TYPE_NUMERICAL_NOTE, + CUDNN_NUMERICAL_NOTE_TYPE_COUNT, + &elem_count, + numeric.data()); + RETURN_CUDNN_FRONTEND_ERROR_IF((status != CUDNN_STATUS_SUCCESS), + error_code_t::HEURISTIC_QUERY_FAILED, + "Heuristic query Numerical Note failed"); + status = detail::get_attribute(extractedEngine_, + CUDNN_ATTR_ENGINE_BEHAVIOR_NOTE, + CUDNN_TYPE_BEHAVIOR_NOTE, + CUDNN_BEHAVIOR_NOTE_TYPE_COUNT, + &elem_count, + nullptr); + RETURN_CUDNN_FRONTEND_ERROR_IF((status != CUDNN_STATUS_SUCCESS), + error_code_t::HEURISTIC_QUERY_FAILED, + "Heuristic query Behavior Note failed"); + + behavior.resize(static_cast(elem_count)); + status = detail::get_attribute(extractedEngine_, + CUDNN_ATTR_ENGINE_BEHAVIOR_NOTE, + CUDNN_TYPE_BEHAVIOR_NOTE, + CUDNN_BEHAVIOR_NOTE_TYPE_COUNT, + &elem_count, + behavior.data()); + RETURN_CUDNN_FRONTEND_ERROR_IF((status != CUDNN_STATUS_SUCCESS), + error_code_t::HEURISTIC_QUERY_FAILED, + "Heuristic query Behavior Note failed"); + + std::vector numerics; + numerics.resize(numeric.size()); + for (auto& note : numeric) { + numerics.push_back(detail::convert_from_cudnn_type(note)); + } + numeric_notes.emplace_back(std::move(numerics)); + + std::vector behaviors; + behaviors.reserve(behaviors.size()); + for (auto& note : behavior) { + behaviors.push_back(detail::convert_from_cudnn_type(note)); + } + behavior_notes.emplace_back(std::move(behaviors)); + } + return {error_code_t::OK, ""}; + } + + error_t + filter_numeric_notes(std::vector const& notes, bool const keep) { + for (auto& note : notes) { + for (auto i = 0u; i < engine_configs.size(); i++) { + bool has_barred_note = + std::find(numeric_notes[i].begin(), numeric_notes[i].end(), note) != numeric_notes[i].end(); + + barred_indices[i] = barred_indices[i] || (has_barred_note ? !keep : keep); + } + } + return {error_code_t::OK, ""}; + } + + error_t + filter_behavior_notes(std::vector const& notes, bool const keep) { + for (auto& note : notes) { + for (auto i = 0u; i < engine_configs.size(); i++) { + bool has_barred_note = + std::find(behavior_notes[i].begin(), behavior_notes[i].end(), note) != behavior_notes[i].end(); + + barred_indices[i] = barred_indices[i] || (has_barred_note ? !keep : keep); + } + } + return {error_code_t::OK, ""}; + } + + void + set_max_workspace_allowed(int64_t const workspace_allowed) { + max_workspace_allowed = workspace_allowed; + } + + void + set_max_shared_mem_allowed(int64_t const smem_allowed) { + max_shared_mem_allowed = smem_allowed; + } + + void + set_barred_names(std::vector const& engine_names) { + barred_engine_names = engine_names; + } + + EngineConfigList + get_barred_engine_configs() { + EngineConfigList barred_engine_configs; + CUDNN_FE_LOG_LABEL_ENDL("INFO: " << " Filtering engine_configs ..." << engine_configs.size()); + for (auto i = 0u; i < engine_configs.size(); i++) { + if (barred_indices[i] == false) { + barred_engine_configs.push_back(engine_configs[i]); + } + } + CUDNN_FE_LOG_LABEL_ENDL("INFO: " << " barred engine_configs ..." << barred_engine_configs.size()); + return barred_engine_configs; + } + + error_t + get_name_at_index(int64_t index, std::string& name) const { + name = detail::get_engine_tag(engine_configs[index]); + return {error_code_t::OK, ""}; + } + + // Structured counterpart of get_name_at_index(): engine index + knob choices, + // suitable for replay via Graph::create_execution_plan(engine_id, knobs). + error_t + get_engine_and_knobs_at_index(int64_t index, + int64_t& engine_id, + std::unordered_map& knobs) const { + RETURN_CUDNN_FRONTEND_ERROR_IF(index < 0 || index >= static_cast(engine_configs.size()), + error_code_t::GRAPH_EXECUTION_FAILED, + "Plan index " + std::to_string(index) + " is invalid."); + std::vector> backend_knobs; + auto status = detail::get_engine_id_and_knobs(engine_configs[index], engine_id, backend_knobs); + RETURN_CUDNN_FRONTEND_ERROR_IF( + status != CUDNN_STATUS_SUCCESS, + error_code_t::CUDNN_BACKEND_API_FAILED, + "Failed to query engine id / knob choices for plan at index " + std::to_string(index)); + knobs.clear(); + for (auto const& backend_knob : backend_knobs) { + knobs[convert_from_backend_knob_type(backend_knob.first)] = backend_knob.second; + } + return {error_code_t::OK, ""}; + } + + error_t + check_support_at_index(int64_t index) { + // Ignore if the engine config was deselected. + // This usually happens when user deselects by numerical and behavioural notes. + + RETURN_CUDNN_FRONTEND_ERROR_IF((index < 0) || (static_cast(barred_indices.size()) <= index), + error_code_t::GRAPH_EXECUTION_FAILED, + "Plan index " + std::to_string(index) + " is invalid."); + + if (barred_indices[index] == true) { + CUDNN_FE_LOG_LABEL_ENDL("Deselecting execution plan at position " << index); + } + + RETURN_CUDNN_FRONTEND_ERROR_IF(barred_indices[index] == true, + error_code_t::GRAPH_EXECUTION_PLAN_CREATION_FAILED, + "Deselecting execution plan"); + + // Ignore if engine name was specified to be ignored by the user. + auto is_blocked = [](std::string const& full_name, std::vector const& blocked_names) -> bool { + for (auto const& blocked_name : blocked_names) { + if (full_name.find(blocked_name) != std::string::npos) { + return true; + } + } + return false; + }; + auto cfg_tag = detail::get_engine_tag(engine_configs[index]); + if (is_blocked(cfg_tag, barred_engine_names)) { + barred_indices[index] = true; + return {error_code_t::GRAPH_EXECUTION_PLAN_CREATION_FAILED, + "[cudnn_frontend] Error: Deselecting execution plan with name " + cfg_tag + " at position " + + std::to_string(index)}; + } + + if (detail::get_backend_version() >= 90200 && detail::get_compiled_version() >= 90200) { + // Ignore kernels that require larger than tolerable shared memory. + int32_t shared_memory_size = INT32_MAX; + auto status = detail::get_shared_memory_size(engine_configs[index], shared_memory_size); + if (status.is_bad()) { + CUDNN_FE_LOG_LABEL_ENDL("WARN: Unknown Shared memory size, so not deselecting plan at position " + << index); + } else if (shared_memory_size > max_shared_mem_allowed) { + barred_indices[index] = true; + return {error_code_t::GRAPH_EXECUTION_PLAN_CREATION_FAILED, + "[cudnn_frontend] Error: Skipping plan since shared memory violation. Requires " + + std::to_string(shared_memory_size)}; + } + + // Filter by workspace can happen at this engine config stage itself. + int64_t workspace_size = INT64_MAX; + CHECK_CUDNN_FRONTEND_ERROR(detail::get_workspace_size(engine_configs[index], workspace_size)); + if (workspace_size > max_workspace_allowed) { + barred_indices[index] = true; + return {error_code_t::GRAPH_EXECUTION_PLAN_CREATION_FAILED, + "[cudnn_frontend] Error: Skipping plan since workspace violation. Requires " + + std::to_string(workspace_size)}; + } + } + // Else we need to build the config. A successful execution plan build means that check_support succeeded. + else { + CHECK_CUDNN_FRONTEND_ERROR(_build_plan_at_index_impl(index)); + } + + CUDNN_FE_LOG_LABEL_ENDL("Check support for index " << index << " passed with cfg " << cfg_tag); + // All checks passed for this config, so return success. + return {error_code_t::OK, ""}; + } + + error_t + check_support() { + // Go over each engine config and return true when you find the first one that is supported. + for (auto i = 0u; i < engine_configs.size(); i++) { + auto status = check_support_at_index(i); + if (status.is_good()) { + return {error_code_t::OK, ""}; + } + } + + std::string err_msg = detail::get_last_error_string_(); + CUDNN_FE_LOG_LABEL_ENDL("ERROR: No valid engine configs returned from heuristics.\n" << err_msg); + return {error_code_t::GRAPH_EXECUTION_PLAN_CREATION_FAILED, + "[cudnn_frontend] Error: No execution plans support the graph." + err_msg}; + } + + error_t + get_behavior_notes_at_index(int64_t const index, std::vector& notes) const { + RETURN_CUDNN_FRONTEND_ERROR_IF((index < 0) || (static_cast(behavior_notes.size()) <= index), + error_code_t::GRAPH_EXECUTION_FAILED, + "Plan index " + std::to_string(index) + " is invalid."); + + notes = behavior_notes[index]; + + return {error_code_t::OK, ""}; + } + + error_t + build_plans(cudnnHandle_t handle, std::string const& json) { + execution_plans.resize(1); + auto const& fe_status = detail::create_cudnn_execution_plan(execution_plans[0], json, handle); + + if (fe_status.is_good()) { + candidate = 0; + } + + return fe_status; + } + + error_t + build_plan_at_index(int64_t index) { + CHECK_CUDNN_FRONTEND_ERROR(check_support_at_index(index)); + CHECK_CUDNN_FRONTEND_ERROR(_build_plan_at_index_impl(index)); + + return {error_code_t::OK, ""}; + } + + error_t + build_plans(BuildPlanPolicy_t const policy, bool const do_multithreaded_builds) { + RETURN_CUDNN_FRONTEND_ERROR_IF(do_multithreaded_builds, + error_code_t::GRAPH_EXECUTION_PLAN_CREATION_FAILED, + "Doing multithreaded builds is not yet supported."); + + // short circuit in case a plan was already created. + // This happens as check_support for v8 builds a plan. + if (policy == BuildPlanPolicy_t::HEURISTICS_CHOICE && candidate != -1) { + return {error_code_t::OK, ""}; + } + + for (auto i = 0u; i < engine_configs.size(); i++) { + auto status = build_plan_at_index(i); + if (status.is_bad()) { + CUDNN_FE_LOG_LABEL_ENDL("WARN: Failed to build plan at " << i); + continue; + } + + // Only set the candidate the first time, as the order of iteration is from highest to lowest priority + if (candidate == -1) { + candidate = static_cast(i); + CUDNN_FE_LOG_LABEL_ENDL("INFO: Candidate set as " << i); + } + + // Return from this function as first successfully built plan is found. + if (policy == BuildPlanPolicy_t::HEURISTICS_CHOICE) { + return {error_code_t::OK, ""}; + } + } + + // Return an error if no execution plans could be built + RETURN_CUDNN_FRONTEND_ERROR_IF(candidate == -1, + error_code_t::GRAPH_EXECUTION_PLAN_CREATION_FAILED, + "[cudnn_frontend] Error: No valid execution plans built."); + + return {error_code_t::OK, ""}; + } + + int64_t + get_autotune_workspace() const { + int64_t max_size = 0; + for (auto& plan : execution_plans) { + max_size = std::max(max_size, plan->getWorkspaceSize()); + } + return max_size; + } + + static error_t + autotune_default_impl(std::vector>& execution_plans, + cudnnHandle_t handle, + std::unordered_map const& tensor_to_pointer_map, + void* workspace_ptr, + void*) { + // Create the variant pack for all the plans to use. + std::vector uids; + std::vector ptrs; + for (auto it : tensor_to_pointer_map) { + uids.push_back(it.first); + ptrs.push_back(it.second); + } + + std::vector> time_sorted_plans; + + auto plan_cmp = [](std::shared_ptr a, std::shared_ptr b) { + return a->getExecutionTime() < b->getExecutionTime(); + }; + + std::multiset, decltype(plan_cmp)> timed_execution_plans(plan_cmp); + + const int maxIterCount = 100; + const float threshhold = 0.95f; + uint64_t successful_plan_count = 0; + cudaEvent_t start, stop; + detail::cuda_event_create(&start); + detail::cuda_event_create(&stop); + detail::cuda_device_synchronize(); + + cudaStream_t stream = nullptr; + detail::get_stream(handle, &stream); + + for (auto plan : execution_plans) { + float time_ms = 0.0f; + float final_time_ms = 0.0f; + float min_time_ms = std::numeric_limits::max(); + + // Warm-up run + CHECK_CUDNN_FRONTEND_ERROR(detail::execute(handle, plan.get(), ptrs, uids, workspace_ptr)); + successful_plan_count++; + detail::cuda_device_synchronize(); + + for (int i = 0; i < maxIterCount; i++) { + detail::cuda_event_record(start, stream); + + auto status = detail::execute(handle, plan.get(), ptrs, uids, workspace_ptr); + + detail::cuda_event_record(stop, stream); + detail::cuda_event_synchronize(stop); + detail::cuda_event_elapsed_time(&time_ms, start, stop); + + final_time_ms = std::min(min_time_ms, time_ms); + if (time_ms / min_time_ms < threshhold) { + min_time_ms = final_time_ms; + } else { + break; + } + } + + CUDNN_FE_LOG_LABEL_ENDL("Plan " << plan->getTag() << " took " << std::setw(10) << final_time_ms); + plan->setExecutionTime(final_time_ms); + timed_execution_plans.insert(plan); + } + + execution_plans.clear(); + for (auto sorted_plan : timed_execution_plans) { + execution_plans.push_back(sorted_plan); + } + + detail::cuda_event_destroy(start); + detail::cuda_event_destroy(stop); + + CUDNN_FE_LOG_LABEL_ENDL("Autotuned " << successful_plan_count << " plans."); + return {error_code_t::OK, ""}; + } + + std::function>&, + cudnnHandle_t, + std::unordered_map const&, + void*, + void*)> + autotune_impl = &Execution_plan_list::autotune_default_impl; + + error_t + autotune(cudnnHandle_t handle, + std::unordered_map const& tensor_to_pointer_map, + void* workspace, + void* user_impl = nullptr) { + auto error = autotune_impl(execution_plans, handle, tensor_to_pointer_map, workspace, user_impl); + return error; + } + + error_t + is_plan_index_executable(int64_t const index) const { + // OSS SDPA engine path + if (index == OSS_SDPA_ENGINE_CANDIDATE) { + RETURN_CUDNN_FRONTEND_ERROR_IF( + !oss_sdpa_engine_built_, error_code_t::GRAPH_EXECUTION_FAILED, "OSS SDPA engine not built."); + return {error_code_t::OK, ""}; + } + + // OSS RmsNorm+SiLU engine path + if (index == OSS_RMS_NORM_SILU_ENGINE_CANDIDATE) { + RETURN_CUDNN_FRONTEND_ERROR_IF( + !oss_rms_norm_silu_built_, error_code_t::GRAPH_EXECUTION_FAILED, "OSS RmsNorm+SiLU engine not built."); + return {error_code_t::OK, ""}; + } + + RETURN_CUDNN_FRONTEND_ERROR_IF((index < 0) || (static_cast(execution_plans.size()) <= index), + error_code_t::GRAPH_EXECUTION_FAILED, + "Plan index " + std::to_string(index) + " is invalid."); + + RETURN_CUDNN_FRONTEND_ERROR_IF(execution_plans[index] == nullptr, + error_code_t::GRAPH_EXECUTION_FAILED, + "Plan index " + std::to_string(index) + " did not build."); + + return {error_code_t::OK, ""}; + } + + // ================================================================ + // Open-source NVRTC engine support + // ================================================================ + + static constexpr int64_t OSS_SDPA_ENGINE_CANDIDATE = -2; + + // Context cached from the Graph for OSS engine execution + struct OssSdpaEngineContext { + int64_t batch = 0, heads_q = 0, heads_kv = 0, seq_q = 0, seq_kv = 0, d = 0; + int64_t q_uid = -1, k_uid = -1, v_uid = -1, o_uid = -1, max_uid = -1, sum_exp_uid = -1; + std::vector q_stride, k_stride, v_stride, o_stride; + std::vector max_stride, sum_exp_stride; + std::optional attn_scale; + // Pre-computed slot indices into the variant pack template (set by prepare_variant_pack_template) + int q_slot = -1, k_slot = -1, v_slot = -1, o_slot = -1, max_slot = -1, sum_exp_slot = -1; + }; + + void + set_oss_sdpa_engine(std::shared_ptr engine) { + oss_sdpa_engine_ = std::move(engine); + } + + void + set_oss_sdpa_engine_context(OssSdpaEngineContext ctx) { + oss_sdpa_ctx_ = std::move(ctx); + } + + bool + has_oss_sdpa_engine() const { + return oss_sdpa_engine_ != nullptr; + } + + bool + is_oss_sdpa_candidate() const { + return candidate == OSS_SDPA_ENGINE_CANDIDATE; + } + + error_t + check_oss_sdpa_engine_support(int64_t sm_version) { + RETURN_CUDNN_FRONTEND_ERROR_IF( + !oss_sdpa_engine_, error_code_t::GRAPH_NOT_SUPPORTED, "No OSS engine registered"); + cudnn_frontend::experimental::AttentionShape_t shape = { + static_cast(oss_sdpa_ctx_.batch), + static_cast(oss_sdpa_ctx_.heads_q), + static_cast(oss_sdpa_ctx_.heads_kv), + static_cast(oss_sdpa_ctx_.heads_kv), + static_cast(oss_sdpa_ctx_.seq_q), + static_cast(oss_sdpa_ctx_.seq_kv), + static_cast(oss_sdpa_ctx_.d), + static_cast(oss_sdpa_ctx_.d), + }; + auto status = oss_sdpa_engine_->check_support(shape, sm_version); + if (status.is_good()) { + oss_sdpa_engine_supported_ = true; + candidate = OSS_SDPA_ENGINE_CANDIDATE; + } + return status; + } + + error_t + build_oss_sdpa_engine() { + RETURN_CUDNN_FRONTEND_ERROR_IF( + !oss_sdpa_engine_supported_, error_code_t::GRAPH_NOT_SUPPORTED, "OSS engine not supported"); + auto status = oss_sdpa_engine_->build(); + if (status.is_good()) { + oss_sdpa_engine_built_ = true; + candidate = OSS_SDPA_ENGINE_CANDIDATE; + } + return status; + } + + // Flat-array execute: takes pre-indexed pointer array (from VariantPackTemplate) + error_t + execute_oss_sdpa_engine(void* const* ptrs, void* workspace, int device, cudaStream_t stream) const { + RETURN_CUDNN_FRONTEND_ERROR_IF( + !oss_sdpa_engine_built_, error_code_t::GRAPH_EXECUTION_FAILED, "OSS engine not built"); + RETURN_CUDNN_FRONTEND_ERROR_IF( + oss_sdpa_ctx_.q_slot < 0 || oss_sdpa_ctx_.k_slot < 0 || oss_sdpa_ctx_.v_slot < 0 || + oss_sdpa_ctx_.o_slot < 0 || oss_sdpa_ctx_.max_slot < 0 || oss_sdpa_ctx_.sum_exp_slot < 0, + error_code_t::INVALID_VARIANT_PACK, + "OSS SDPA slot indices not initialized. Call prepare_variant_pack_template() first."); + + void* q_ptr = ptrs[oss_sdpa_ctx_.q_slot]; + void* k_ptr = ptrs[oss_sdpa_ctx_.k_slot]; + void* v_ptr = ptrs[oss_sdpa_ctx_.v_slot]; + void* o_ptr = ptrs[oss_sdpa_ctx_.o_slot]; + void* max_ptr = ptrs[oss_sdpa_ctx_.max_slot]; + void* sum_exp_ptr = ptrs[oss_sdpa_ctx_.sum_exp_slot]; + + RETURN_CUDNN_FRONTEND_ERROR_IF(!q_ptr || !k_ptr || !v_ptr || !o_ptr, + error_code_t::INVALID_VARIANT_PACK, + "Missing Q/K/V/O pointers for OSS engine"); + RETURN_CUDNN_FRONTEND_ERROR_IF(!max_ptr || !sum_exp_ptr, + error_code_t::INVALID_VARIANT_PACK, + "Missing max/sum_exp pointers for OSS engine"); + + return oss_sdpa_engine_->execute(static_cast(oss_sdpa_ctx_.batch), + static_cast(oss_sdpa_ctx_.heads_q), + static_cast(oss_sdpa_ctx_.heads_kv), + static_cast(oss_sdpa_ctx_.seq_q), + static_cast(oss_sdpa_ctx_.seq_kv), + static_cast(oss_sdpa_ctx_.d), + q_ptr, + oss_sdpa_ctx_.q_stride, + k_ptr, + oss_sdpa_ctx_.k_stride, + v_ptr, + oss_sdpa_ctx_.v_stride, + o_ptr, + oss_sdpa_ctx_.o_stride, + max_ptr, + oss_sdpa_ctx_.max_stride, + sum_exp_ptr, + oss_sdpa_ctx_.sum_exp_stride, + workspace, + device, + stream, + oss_sdpa_ctx_.attn_scale); + } + + // Flat-array overload with dynamic shape overrides + error_t + execute_oss_sdpa_engine(void* const* ptrs, + void* workspace, + int device, + cudaStream_t stream, + std::vector const& override_uids, + std::vector> const& override_shapes, + std::vector> const& override_strides) const { + RETURN_CUDNN_FRONTEND_ERROR_IF( + !oss_sdpa_engine_built_, error_code_t::GRAPH_EXECUTION_FAILED, "OSS engine not built"); + RETURN_CUDNN_FRONTEND_ERROR_IF( + override_uids.size() != override_shapes.size() || override_uids.size() != override_strides.size(), + error_code_t::INVALID_VALUE, + "override_uids/shapes/strides must have the same size"); + + // Build uid → index lookup for overrides + std::unordered_map uid_to_idx; + for (size_t i = 0; i < override_uids.size(); ++i) { + uid_to_idx[override_uids[i]] = i; + } + auto resolve_shape = [&](int64_t uid, std::vector const& def) -> std::vector const& { + auto it = uid_to_idx.find(uid); + return (it != uid_to_idx.end()) ? override_shapes[it->second] : def; + }; + auto resolve_stride = [&](int64_t uid, std::vector const& def) -> std::vector const& { + auto it = uid_to_idx.find(uid); + return (it != uid_to_idx.end()) ? override_strides[it->second] : def; + }; + + // Resolve shapes + std::vector q_def = {oss_sdpa_ctx_.batch, oss_sdpa_ctx_.heads_q, oss_sdpa_ctx_.seq_q, oss_sdpa_ctx_.d}; + auto const& q_shape = resolve_shape(oss_sdpa_ctx_.q_uid, q_def); + auto const& q_stride = resolve_stride(oss_sdpa_ctx_.q_uid, oss_sdpa_ctx_.q_stride); + RETURN_CUDNN_FRONTEND_ERROR_IF(q_shape.size() < 4, error_code_t::INVALID_VALUE, "Q shape must have >=4 dims"); + int64_t batch = q_shape[0], heads_q = q_shape[1], seq_q = q_shape[2], d = q_shape[3]; + RETURN_CUDNN_FRONTEND_ERROR_IF(d != oss_sdpa_ctx_.d, + error_code_t::INVALID_VALUE, + "Cannot change d dynamically (compiled=" + std::to_string(oss_sdpa_ctx_.d) + + ", got=" + std::to_string(d) + ")"); + + std::vector k_def = { + oss_sdpa_ctx_.batch, oss_sdpa_ctx_.heads_kv, oss_sdpa_ctx_.seq_kv, oss_sdpa_ctx_.d}; + auto const& k_shape = resolve_shape(oss_sdpa_ctx_.k_uid, k_def); + auto const& k_stride = resolve_stride(oss_sdpa_ctx_.k_uid, oss_sdpa_ctx_.k_stride); + int64_t heads_kv = k_shape[1], seq_kv = k_shape[2]; + + auto const& v_stride = resolve_stride(oss_sdpa_ctx_.v_uid, oss_sdpa_ctx_.v_stride); + auto const& o_stride = resolve_stride(oss_sdpa_ctx_.o_uid, oss_sdpa_ctx_.o_stride); + auto const& max_stride = resolve_stride(oss_sdpa_ctx_.max_uid, oss_sdpa_ctx_.max_stride); + auto const& se_stride = resolve_stride(oss_sdpa_ctx_.sum_exp_uid, oss_sdpa_ctx_.sum_exp_stride); + + // Validate slot indices + RETURN_CUDNN_FRONTEND_ERROR_IF( + oss_sdpa_ctx_.q_slot < 0 || oss_sdpa_ctx_.k_slot < 0 || oss_sdpa_ctx_.v_slot < 0 || + oss_sdpa_ctx_.o_slot < 0 || oss_sdpa_ctx_.max_slot < 0 || oss_sdpa_ctx_.sum_exp_slot < 0, + error_code_t::INVALID_VARIANT_PACK, + "OSS SDPA slot indices not initialized. Call prepare_variant_pack_template() first."); + + // Pointers from pre-indexed slots + void* q_ptr = ptrs[oss_sdpa_ctx_.q_slot]; + void* k_ptr = ptrs[oss_sdpa_ctx_.k_slot]; + void* v_ptr = ptrs[oss_sdpa_ctx_.v_slot]; + void* o_ptr = ptrs[oss_sdpa_ctx_.o_slot]; + void* max_ptr = ptrs[oss_sdpa_ctx_.max_slot]; + void* sum_exp_ptr = ptrs[oss_sdpa_ctx_.sum_exp_slot]; + + RETURN_CUDNN_FRONTEND_ERROR_IF( + !q_ptr || !k_ptr || !v_ptr || !o_ptr, error_code_t::INVALID_VARIANT_PACK, "Missing Q/K/V/O pointers"); + RETURN_CUDNN_FRONTEND_ERROR_IF( + !max_ptr || !sum_exp_ptr, error_code_t::INVALID_VARIANT_PACK, "Missing max/sum_exp pointers"); + + return oss_sdpa_engine_->execute(static_cast(batch), + static_cast(heads_q), + static_cast(heads_kv), + static_cast(seq_q), + static_cast(seq_kv), + static_cast(d), + q_ptr, + q_stride, + k_ptr, + k_stride, + v_ptr, + v_stride, + o_ptr, + o_stride, + max_ptr, + max_stride, + sum_exp_ptr, + se_stride, + workspace, + device, + stream, + oss_sdpa_ctx_.attn_scale); + } + + // ================================================================ + // Open-source NVRTC engine support: RmsNorm + SiLU + // ================================================================ + + static constexpr int64_t OSS_RMS_NORM_SILU_ENGINE_CANDIDATE = -3; + + // Context cached from the Graph for RmsNorm+SiLU engine execution + struct OssRmsNormSiluContext { + int64_t num_tokens = 0; + int64_t C = 0; + + // Tensor UIDs for pointer lookup from variant pack + int64_t x_uid = -1; // input [num_tokens, C] + int64_t y_uid = -1; // output [num_tokens, C] (after SiLU) + int64_t scale_uid = -1; // gamma weights [C] + int64_t bias_uid = -1; // beta bias [C], optional (-1 if absent) + int64_t epsilon_uid = -1; // epsilon scalar + + // FP8 output: optional scale/scale_inv/amax tensor UIDs + int64_t fp8_scale_uid = -1; + int64_t fp8_scale_inv_uid = -1; + int64_t fp8_amax_uid = -1; + + // NVFP4 output: row-wise block-scale tensor UID + int64_t nvfp4_scale_row_uid = -1; + + experimental::RmsNormSiluDtype output_dtype = experimental::RmsNormSiluDtype::BF16; + + // Pre-computed slot indices into the variant pack template + int x_slot = -1, y_slot = -1, scale_slot = -1, bias_slot = -1, epsilon_slot = -1; + int fp8_scale_slot = -1, fp8_scale_inv_slot = -1, fp8_amax_slot = -1; + int nvfp4_scale_row_slot = -1; + }; + + void + set_oss_rms_norm_silu_engine(std::shared_ptr engine) { + oss_rms_norm_silu_engine_ = std::move(engine); + } + + void + set_oss_rms_norm_silu_context(OssRmsNormSiluContext ctx) { + oss_rms_norm_silu_ctx_ = std::move(ctx); + } + + bool + has_oss_rms_norm_silu_engine() const { + return oss_rms_norm_silu_engine_ != nullptr; + } + + bool + is_oss_rms_norm_silu_candidate() const { + return candidate == OSS_RMS_NORM_SILU_ENGINE_CANDIDATE; + } + + error_t + check_oss_rms_norm_silu_support(int64_t sm_version) { + RETURN_CUDNN_FRONTEND_ERROR_IF( + !oss_rms_norm_silu_engine_, error_code_t::GRAPH_NOT_SUPPORTED, "No RmsNorm+SiLU OSS engine registered"); + experimental::NormSiluShape_t shape; + shape.C = static_cast(oss_rms_norm_silu_ctx_.C); + shape.num_tokens = static_cast(oss_rms_norm_silu_ctx_.num_tokens); + shape.output_dtype = oss_rms_norm_silu_ctx_.output_dtype; + auto status = oss_rms_norm_silu_engine_->check_support(shape, static_cast(sm_version)); + if (status.is_good()) { + oss_rms_norm_silu_supported_ = true; + candidate = OSS_RMS_NORM_SILU_ENGINE_CANDIDATE; + } + return status; + } + + error_t + build_oss_rms_norm_silu_engine() { + RETURN_CUDNN_FRONTEND_ERROR_IF( + !oss_rms_norm_silu_supported_, error_code_t::GRAPH_NOT_SUPPORTED, "RmsNorm+SiLU OSS engine not supported"); + auto status = oss_rms_norm_silu_engine_->build(); + if (status.is_good()) { + oss_rms_norm_silu_built_ = true; + candidate = OSS_RMS_NORM_SILU_ENGINE_CANDIDATE; + } + return status; + } + + int64_t + get_oss_rms_norm_silu_workspace_size() const { + if (!oss_rms_norm_silu_engine_) return 0; + return oss_rms_norm_silu_engine_->get_workspace_size(); + } + + // Set pre-computed slot indices for OSS engines (called by prepare_variant_pack_template) + void + set_oss_slot_indices(std::function const& slot_for) { + if (is_oss_sdpa_candidate()) { + oss_sdpa_ctx_.q_slot = slot_for(oss_sdpa_ctx_.q_uid); + oss_sdpa_ctx_.k_slot = slot_for(oss_sdpa_ctx_.k_uid); + oss_sdpa_ctx_.v_slot = slot_for(oss_sdpa_ctx_.v_uid); + oss_sdpa_ctx_.o_slot = slot_for(oss_sdpa_ctx_.o_uid); + oss_sdpa_ctx_.max_slot = slot_for(oss_sdpa_ctx_.max_uid); + oss_sdpa_ctx_.sum_exp_slot = slot_for(oss_sdpa_ctx_.sum_exp_uid); + } + if (is_oss_rms_norm_silu_candidate()) { + oss_rms_norm_silu_ctx_.x_slot = slot_for(oss_rms_norm_silu_ctx_.x_uid); + oss_rms_norm_silu_ctx_.y_slot = slot_for(oss_rms_norm_silu_ctx_.y_uid); + oss_rms_norm_silu_ctx_.scale_slot = slot_for(oss_rms_norm_silu_ctx_.scale_uid); + oss_rms_norm_silu_ctx_.bias_slot = slot_for(oss_rms_norm_silu_ctx_.bias_uid); + oss_rms_norm_silu_ctx_.epsilon_slot = slot_for(oss_rms_norm_silu_ctx_.epsilon_uid); + oss_rms_norm_silu_ctx_.fp8_scale_slot = slot_for(oss_rms_norm_silu_ctx_.fp8_scale_uid); + oss_rms_norm_silu_ctx_.fp8_scale_inv_slot = slot_for(oss_rms_norm_silu_ctx_.fp8_scale_inv_uid); + oss_rms_norm_silu_ctx_.fp8_amax_slot = slot_for(oss_rms_norm_silu_ctx_.fp8_amax_uid); + oss_rms_norm_silu_ctx_.nvfp4_scale_row_slot = slot_for(oss_rms_norm_silu_ctx_.nvfp4_scale_row_uid); + } + } + + // Flat-array overload for RmsNorm+SiLU + error_t + execute_oss_rms_norm_silu_engine(void* const* ptrs, void* workspace, int device, cudaStream_t stream) const { + RETURN_CUDNN_FRONTEND_ERROR_IF( + !oss_rms_norm_silu_built_, error_code_t::GRAPH_EXECUTION_FAILED, "RmsNorm+SiLU OSS engine not built"); + + auto const& ctx = oss_rms_norm_silu_ctx_; + RETURN_CUDNN_FRONTEND_ERROR_IF( + ctx.x_slot < 0 || ctx.y_slot < 0 || ctx.scale_slot < 0 || ctx.epsilon_slot < 0, + error_code_t::INVALID_VARIANT_PACK, + "OSS RmsNorm+SiLU slot indices not initialized. Call prepare_variant_pack_template() first."); + + void* x_ptr = ptrs[ctx.x_slot]; + void* y_ptr = ptrs[ctx.y_slot]; + void* scale_ptr = ptrs[ctx.scale_slot]; + void* bias_ptr = (ctx.bias_slot >= 0) ? ptrs[ctx.bias_slot] : nullptr; + void* eps_ptr = ptrs[ctx.epsilon_slot]; + + RETURN_CUDNN_FRONTEND_ERROR_IF(!x_ptr || !y_ptr || !scale_ptr, + error_code_t::INVALID_VARIANT_PACK, + "Missing X/Y/SCALE pointers for RmsNorm+SiLU OSS engine"); + RETURN_CUDNN_FRONTEND_ERROR_IF( + !eps_ptr, error_code_t::INVALID_VARIANT_PACK, "Missing EPSILON pointer for RmsNorm+SiLU OSS engine"); + + float epsilon = *static_cast(eps_ptr); + + experimental::RmsNormSiluExtraParams extra; + auto slot_ptr = [&](int slot) -> void* { return (slot >= 0) ? ptrs[slot] : nullptr; }; + extra.fp8_scale = slot_ptr(ctx.fp8_scale_slot); + extra.fp8_scale_inv = slot_ptr(ctx.fp8_scale_inv_slot); + extra.fp8_amax = slot_ptr(ctx.fp8_amax_slot); + extra.nvfp4_scale_row = slot_ptr(ctx.nvfp4_scale_row_slot); + + return oss_rms_norm_silu_engine_->execute(x_ptr, + y_ptr, + scale_ptr, + bias_ptr, + static_cast(ctx.num_tokens), + static_cast(ctx.C), + epsilon, + workspace, + device, + stream, + extra); + } + + private: + std::shared_ptr oss_sdpa_engine_; + bool oss_sdpa_engine_supported_ = false; + bool oss_sdpa_engine_built_ = false; + OssSdpaEngineContext oss_sdpa_ctx_; + + std::shared_ptr oss_rms_norm_silu_engine_; + bool oss_rms_norm_silu_supported_ = false; + bool oss_rms_norm_silu_built_ = false; + OssRmsNormSiluContext oss_rms_norm_silu_ctx_; +}; + +} // namespace graph +} // namespace cudnn_frontend diff --git a/deps/cudnn-frontend/include/cudnn_frontend/thirdparty/nlohmann/LICENSE.MIT b/deps/cudnn-frontend/include/cudnn_frontend/thirdparty/nlohmann/LICENSE.MIT new file mode 100644 index 0000000..1c1f7a6 --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/thirdparty/nlohmann/LICENSE.MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2013-2022 Niels Lohmann + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/deps/cudnn-frontend/include/cudnn_frontend/thirdparty/nlohmann/json.hpp b/deps/cudnn-frontend/include/cudnn_frontend/thirdparty/nlohmann/json.hpp new file mode 100644 index 0000000..85b4cdd --- /dev/null +++ b/deps/cudnn-frontend/include/cudnn_frontend/thirdparty/nlohmann/json.hpp @@ -0,0 +1,26710 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.3 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann +// SPDX-License-Identifier: MIT + +/****************************************************************************\ + * Note on documentation: The source files contain links to the online * + * documentation of the public API at https://json.nlohmann.me. This URL * + * contains the most recent documentation and should also be applicable to * + * previous versions; documentation for deprecated functions is not * + * removed, but marked deprecated. See "Generate documentation" section in * + * file docs/README.md. * +\****************************************************************************/ + +#ifndef INCLUDE_NLOHMANN_JSON_HPP_ +#define INCLUDE_NLOHMANN_JSON_HPP_ + +#include // all_of, find, for_each +#include // nullptr_t, ptrdiff_t, size_t +#include // hash, less +#include // initializer_list +#ifndef JSON_NO_IO +#include // istream, ostream +#endif // JSON_NO_IO +#include // random_access_iterator_tag +#include // unique_ptr +#include // string, stoi, to_string +#include // declval, forward, move, pair, swap +#include // vector + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.3 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.3 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann +// SPDX-License-Identifier: MIT + +// This file contains all macro definitions affecting or depending on the ABI + +#ifndef JSON_SKIP_LIBRARY_VERSION_CHECK +#if defined(NLOHMANN_JSON_VERSION_MAJOR) && defined(NLOHMANN_JSON_VERSION_MINOR) && defined(NLOHMANN_JSON_VERSION_PATCH) +#if NLOHMANN_JSON_VERSION_MAJOR != 3 || NLOHMANN_JSON_VERSION_MINOR != 11 || NLOHMANN_JSON_VERSION_PATCH != 3 +#warning "Already included a different version of the library!" +#endif +#endif +#endif + +#define NLOHMANN_JSON_VERSION_MAJOR 3 // NOLINT(modernize-macro-to-enum) +#define NLOHMANN_JSON_VERSION_MINOR 11 // NOLINT(modernize-macro-to-enum) +#define NLOHMANN_JSON_VERSION_PATCH 3 // NOLINT(modernize-macro-to-enum) + +#ifndef JSON_DIAGNOSTICS +#define JSON_DIAGNOSTICS 0 +#endif + +#ifndef JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON +#define JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON 0 +#endif + +#if JSON_DIAGNOSTICS +#define NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS _diag +#else +#define NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS +#endif + +#if JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON +#define NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON _ldvcmp +#else +#define NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON +#endif + +#ifndef NLOHMANN_JSON_NAMESPACE_NO_VERSION +#define NLOHMANN_JSON_NAMESPACE_NO_VERSION 0 +#endif + +// Construct the namespace ABI tags component +#define NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b) json_abi##a##b +#define NLOHMANN_JSON_ABI_TAGS_CONCAT(a, b) NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b) + +#define NLOHMANN_JSON_ABI_TAGS \ + NLOHMANN_JSON_ABI_TAGS_CONCAT(NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS, \ + NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON) + +// Construct the namespace version component +#define NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT_EX(major, minor, patch) _v##major##_##minor##_##patch +#define NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT(major, minor, patch) \ + NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT_EX(major, minor, patch) + +#if NLOHMANN_JSON_NAMESPACE_NO_VERSION +#define NLOHMANN_JSON_NAMESPACE_VERSION +#else +#define NLOHMANN_JSON_NAMESPACE_VERSION \ + NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT( \ + NLOHMANN_JSON_VERSION_MAJOR, NLOHMANN_JSON_VERSION_MINOR, NLOHMANN_JSON_VERSION_PATCH) +#endif + +// Combine namespace components +#define NLOHMANN_JSON_NAMESPACE_CONCAT_EX(a, b) a##b +#define NLOHMANN_JSON_NAMESPACE_CONCAT(a, b) NLOHMANN_JSON_NAMESPACE_CONCAT_EX(a, b) + +#ifndef NLOHMANN_JSON_NAMESPACE +#define NLOHMANN_JSON_NAMESPACE \ + nlohmann::NLOHMANN_JSON_NAMESPACE_CONCAT(NLOHMANN_JSON_ABI_TAGS, NLOHMANN_JSON_NAMESPACE_VERSION) +#endif + +#ifndef NLOHMANN_JSON_NAMESPACE_BEGIN +#define NLOHMANN_JSON_NAMESPACE_BEGIN \ + namespace nlohmann { \ + inline namespace NLOHMANN_JSON_NAMESPACE_CONCAT(NLOHMANN_JSON_ABI_TAGS, NLOHMANN_JSON_NAMESPACE_VERSION) { +#endif + +#ifndef NLOHMANN_JSON_NAMESPACE_END +#define NLOHMANN_JSON_NAMESPACE_END \ + } /* namespace (inline namespace) NOLINT(readability/namespace) */ \ + } // namespace nlohmann +#endif + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.3 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include // transform +#include // array +#include // forward_list +#include // inserter, front_inserter, end +#include // map +#include // string +#include // tuple, make_tuple +#include // is_arithmetic, is_same, is_enum, underlying_type, is_convertible +#include // unordered_map +#include // pair, declval +#include // valarray + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.3 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include // nullptr_t +#include // exception +#if JSON_DIAGNOSTICS +#include // accumulate +#endif +#include // runtime_error +#include // to_string +#include // vector + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.3 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include // array +#include // size_t +#include // uint8_t +#include // string + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.3 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include // declval, pair +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.3 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.3 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann +// SPDX-License-Identifier: MIT + +// #include + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail { + +template +struct make_void { + using type = void; +}; +template +using void_t = typename make_void::type; + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail { + +// https://en.cppreference.com/w/cpp/experimental/is_detected +struct nonesuch { + nonesuch() = delete; + ~nonesuch() = delete; + nonesuch(nonesuch const&) = delete; + nonesuch(nonesuch const&&) = delete; + void + operator=(nonesuch const&) = delete; + void + operator=(nonesuch&&) = delete; +}; + +template class Op, class... Args> +struct detector { + using value_t = std::false_type; + using type = Default; +}; + +template class Op, class... Args> +struct detector>, Op, Args...> { + using value_t = std::true_type; + using type = Op; +}; + +template