aboutsummaryrefslogtreecommitdiff
path: root/deps/cudnn-frontend/include/cudnn_frontend/backend/kernel_cache.h
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-07-02 19:44:08 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-07-02 19:44:08 +0200
commit6a10d3577232f1fd646a20f22a4a1ab4ec29f28a (patch)
treee4a077b064876ddb39144935fa256a4ab0692488 /deps/cudnn-frontend/include/cudnn_frontend/backend/kernel_cache.h
parent98a713e86c23bc6098bf7a5558507a9f27a7a1c1 (diff)
downloadffwd-6a10d3577232f1fd646a20f22a4a1ab4ec29f28a.tar.gz
ffwd-6a10d3577232f1fd646a20f22a4a1ab4ec29f28a.zip
cuda: add cuDNN fused attention path with runtime fallback
Diffstat (limited to 'deps/cudnn-frontend/include/cudnn_frontend/backend/kernel_cache.h')
-rw-r--r--deps/cudnn-frontend/include/cudnn_frontend/backend/kernel_cache.h168
1 files changed, 168 insertions, 0 deletions
diff --git a/deps/cudnn-frontend/include/cudnn_frontend/backend/kernel_cache.h b/deps/cudnn-frontend/include/cudnn_frontend/backend/kernel_cache.h
new file mode 100644
index 0000000..ef173b2
--- /dev/null
+++ b/deps/cudnn-frontend/include/cudnn_frontend/backend/kernel_cache.h
@@ -0,0 +1,168 @@
+/*
+ * Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#pragma once
+
+#include <algorithm>
+#include <array>
+#include <functional>
+#include <memory>
+#include <sstream>
+#include <utility>
+
+#include "../graph_helpers.h"
+#include "backend_descriptor.h"
+
+namespace cudnn_frontend {
+namespace graph {
+class Graph;
+} // namespace graph
+///
+/// KernelCache Class
+/// Wraps the kernel_cache backend descriptor
+/// Wraps backend utility functions for user's convenience
+/// Backend accessor functions: size()
+/// Contains internal utilities for kernel cache finalization and operation graph attributes
+///
+class KernelCache : public detail::backend_descriptor {
+ public:
+ friend class graph::Graph;
+ // Uses the default backend constructor so that we can check for initialization error during build()
+ KernelCache() : backend_descriptor() {}
+
+ std::string
+ describe() const {
+ std::stringstream ss;
+ ss << "CUDNN_BACKEND_KERNEL_CACHE_DESCRIPTOR : " << std::endl;
+ return ss.str();
+ }
+
+ bool
+ is_finalized() {
+ return finalized;
+ }
+
+ // Used to check kernel cache status (particularly after initialization)
+ error_t
+ status() {
+ if (get_status() != CUDNN_STATUS_SUCCESS) {
+ return {error_code_t::CUDNN_BACKEND_API_FAILED,
+ "CUDNN_BACKEND_KERNEL_CACHE_DESCRIPTOR: Check CUDNN_VERSION >= 9.4"};
+ }
+ return {};
+ }
+
+ error_t
+ to_json(std::string &str_json) const {
+ str_json.clear();
+#if (CUDNN_VERSION >= 91000)
+ RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 91000,
+ error_code_t::CUDNN_BACKEND_API_FAILED,
+ "CUDNN_ATTR_KERNEL_CACHE_JSON_REPRESENTATION is only available starting 9.10.");
+
+ int64_t serializationSize;
+ std::vector<char> serialization_buf;
+ _CUDNN_CHECK_CUDNN_ERROR(detail::get_attribute(
+ get_ptr(), CUDNN_ATTR_KERNEL_CACHE_JSON_REPRESENTATION, CUDNN_TYPE_CHAR, 0, &serializationSize, nullptr));
+ serialization_buf.resize(static_cast<size_t>(serializationSize));
+
+ _CUDNN_CHECK_CUDNN_ERROR(detail::get_attribute(get_ptr(),
+ CUDNN_ATTR_KERNEL_CACHE_JSON_REPRESENTATION,
+ CUDNN_TYPE_CHAR,
+ serializationSize,
+ &serializationSize,
+ serialization_buf.data()));
+ std::string json_string(serialization_buf.begin(), serialization_buf.end());
+ str_json = std::move(json_string);
+ return {};
+#else
+ (void)str_json;
+ return {error_code_t::CUDNN_BACKEND_API_FAILED,
+ "CUDNN_ATTR_KERNEL_CACHE_JSON_REPRESENTATION is only available starting 9.10."};
+#endif
+ }
+
+ error_t
+ from_json(const std::string &json_cache) {
+#if (CUDNN_VERSION >= 91000)
+ RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 91000,
+ error_code_t::CUDNN_BACKEND_API_FAILED,
+ "CUDNN_ATTR_KERNEL_CACHE_JSON_REPRESENTATION is only available starting 9.10.");
+
+ // Check if the kernel cache is already initialized
+ RETURN_CUDNN_FRONTEND_ERROR_IF(
+ get_ptr() != nullptr, error_code_t::CUDNN_BACKEND_API_FAILED, "Kernel cache is already initialized.");
+
+ // // Initialize the kernel cache descriptor
+ CHECK_CUDNN_FRONTEND_ERROR(initialize(CUDNN_BACKEND_KERNEL_CACHE_DESCRIPTOR));
+
+ std::vector<char> serialization_buf;
+ serialization_buf.assign(json_cache.begin(), json_cache.end());
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(get_ptr(),
+ CUDNN_ATTR_KERNEL_CACHE_JSON_REPRESENTATION,
+ CUDNN_TYPE_CHAR,
+ serialization_buf.size(),
+ serialization_buf.data()));
+ return {};
+#else
+ (void)json_cache;
+ return {error_code_t::CUDNN_BACKEND_API_FAILED,
+ "CUDNN_ATTR_KERNEL_CACHE_JSON_REPRESENTATION is only available starting 9.10."};
+#endif
+ }
+
+ // Responsible for initializing, setting operation graph attribute, and finalizing kernel cache
+ // Check for both compile-time and runtime cuDNN version
+ error_t
+ build(cudnnBackendDescriptor_t op_graph) {
+#if (CUDNN_VERSION >= 90400)
+ RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 90400,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "CUDNN_BACKEND_KERNEL_CACHE_DESCRIPTOR is only available starting 9.4.");
+ if (get_ptr() == nullptr) {
+ CHECK_CUDNN_FRONTEND_ERROR(initialize(CUDNN_BACKEND_KERNEL_CACHE_DESCRIPTOR));
+ }
+#if (CUDNN_VERSION >= 90500)
+ RETURN_CUDNN_FRONTEND_ERROR_IF(detail::get_backend_version() < 90500,
+ error_code_t::GRAPH_NOT_SUPPORTED,
+ "CUDNN_ATTR_KERNEL_CACHE_OPERATION_GRAPH is only available starting 9.5.");
+ if (op_graph) {
+ _CUDNN_CHECK_CUDNN_ERROR(detail::set_attribute(
+ get_ptr(), CUDNN_ATTR_KERNEL_CACHE_OPERATION_GRAPH, CUDNN_TYPE_BACKEND_DESCRIPTOR, 1, &op_graph));
+ }
+#else
+ (void)op_graph;
+#endif
+ CHECK_CUDNN_FRONTEND_ERROR(finalize());
+ finalized = true;
+ return {};
+#else
+ (void)op_graph;
+ return {error_code_t::CUDNN_BACKEND_API_FAILED,
+ "CUDNN_BACKEND_KERNEL_CACHE_DESCRIPTOR is only available starting 9.4."};
+#endif
+ }
+
+ private:
+ bool finalized = false;
+};
+} // namespace cudnn_frontend \ No newline at end of file