CUDNN Frontend API  8.3.0
cudnn_frontend_Heuristics.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20  * DEALINGS IN THE SOFTWARE.
21  */
22 
23 #pragma once
24 
25 #include <vector>
26 
27 #include <cudnn.h>
28 #include <cudnn_backend.h>
29 
32 #include "cudnn_frontend_utils.h"
33 #include "cudnn_frontend_Filters.h"
34 
35 namespace cudnn_frontend {
48  public:
50  std::string
51  describe() const override {
52  std::stringstream ss;
53  ss << "CUDNN_BACKEND_ENGINEHEUR_DESCRIPTOR :";
54  return ss.str();
55  }
56 
57  EngineHeuristics_v8(EngineHeuristics_v8 &&from) = default;
59  operator= (EngineHeuristics_v8 &&from) = default;
60 
61  ~EngineHeuristics_v8() = default;
62 
67  auto
69  getEngineConfig(int64_t count = 1) -> std::vector<ManagedOpaqueDescriptor> & {
70  cudnnStatus_t status;
71  for (auto i = 0u; i < count; ++i) {
72  ManagedOpaqueDescriptor engConfig = nullptr;
73  engConfig = make_shared_backend_pointer(CUDNN_BACKEND_ENGINECFG_DESCRIPTOR);
74  if (engConfig->is_good() == false) {
76  this,
77  engConfig->get_status(),
78  "CUDNN_BACKEND_ENGINEHEUR_DESCRIPTOR: CUDNN_BACKEND_ENGINECFG_DESCRIPTOR cudnnCreate Failed");
79  return m_heuristic_results;
80  };
81  m_heuristic_results.emplace_back(engConfig);
82  }
83  std::vector<cudnnBackendDescriptor_t> heuristic_results_;
84  for (std::uint32_t i = 0; i < m_heuristic_results.size(); i++) {
85  heuristic_results_.emplace_back(m_heuristic_results[i]->get_backend_descriptor());
86  }
87  int64_t result = -1;
88  status = cudnnBackendGetAttribute(pointer->get_backend_descriptor(),
89  CUDNN_ATTR_ENGINEHEUR_RESULTS,
90  CUDNN_TYPE_BACKEND_DESCRIPTOR,
91  count,
92  &result,
93  heuristic_results_.data());
94  if (status != CUDNN_STATUS_SUCCESS) {
96  this, status, "CUDNN_BACKEND_ENGINEHEUR_DESCRIPTOR: GetAttribute CUDNN_ATTR_ENGINEHEUR_RESULTS Failed");
97  };
98  return m_heuristic_results;
99  }
100 
102  auto
103  getEngineConfigCount(void) const -> int64_t {
104  cudnnStatus_t status;
105  int64_t count = -1;
106  status = cudnnBackendGetAttribute(pointer->get_backend_descriptor(),
107  CUDNN_ATTR_ENGINEHEUR_RESULTS,
108  CUDNN_TYPE_BACKEND_DESCRIPTOR,
109  0,
110  &count,
111  nullptr);
112  if (status != CUDNN_STATUS_SUCCESS) {
114  this,
115  status,
116  "CUDNN_BACKEND_ENGINEHEUR_DESCRIPTOR: GetAttribute CUDNN_ATTR_ENGINEHEUR_RESULTS Count Failed");
117  };
118  return count;
119  }
122  private:
123  EngineHeuristics_v8() = default;
124  EngineHeuristics_v8(EngineHeuristics_v8 const &) = delete;
126  operator=(EngineHeuristics_v8 const &) = delete;
127 
128  cudnnBackendHeurMode_t mode = CUDNN_HEUR_MODE_INSTANT;
130  std::vector<ManagedOpaqueDescriptor> m_heuristic_results;
131  std::string opGraphTag;
132 };
133 
138  public:
143  auto
146  m_heuristics.opGraph = opGraph_.get_desc();
147  m_heuristics.opGraphTag = opGraph_.getTag();
148  return *this;
149  }
151  auto
152  setHeurMode(cudnnBackendHeurMode_t mode_) -> EngineHeuristicsBuilder_v8 & {
153  m_heuristics.mode = mode_;
154  return *this;
155  }
161  build() {
162  if (m_heuristics.opGraph == nullptr) {
163  set_error_and_throw_exception(&m_heuristics,
164  CUDNN_STATUS_BAD_PARAM,
165  "CUDNN_BACKEND_ENGINEHEUR_DESCRIPTOR: Check and Set the "
166  "CUDNN_ATTR_ENGINEHEUR_OPERATION_GRAPH field for heuristic");
167  return std::move(m_heuristics);
168  };
169 
170  // Create a descriptor. Memory allocation happens here.
171  auto status = m_heuristics.initialize_managed_backend_pointer(CUDNN_BACKEND_ENGINEHEUR_DESCRIPTOR);
172  if (status != CUDNN_STATUS_SUCCESS) {
174  &m_heuristics, status, "CUDNN_BACKEND_ENGINEHEUR_DESCRIPTOR: cudnnCreate Failed");
175  return std::move(m_heuristics);
176  };
177 
178  status = cudnnBackendSetAttribute(m_heuristics.pointer->get_backend_descriptor(),
179  CUDNN_ATTR_ENGINEHEUR_OPERATION_GRAPH,
180  CUDNN_TYPE_BACKEND_DESCRIPTOR,
181  1,
182  &(m_heuristics.opGraph->get_backend_descriptor()));
183  if (status != CUDNN_STATUS_SUCCESS) {
185  &m_heuristics,
186  status,
187  "CUDNN_BACKEND_ENGINEHEUR_DESCRIPTOR: SetAttribute CUDNN_ATTR_ENGINEHEUR_OPERATION_GRAPH Failed");
188  return std::move(m_heuristics);
189  };
190  status = cudnnBackendSetAttribute(m_heuristics.pointer->get_backend_descriptor(),
191  CUDNN_ATTR_ENGINEHEUR_MODE,
192  CUDNN_TYPE_HEUR_MODE,
193  1,
194  &m_heuristics.mode);
195  if (status != CUDNN_STATUS_SUCCESS) {
197  &m_heuristics,
198  status,
199  "CUDNN_BACKEND_ENGINEHEUR_DESCRIPTOR: SetAttribute CUDNN_ATTR_ENGINEHEUR_MODE Failed");
200  return std::move(m_heuristics);
201  };
202 
203  // Finalizing the descriptor
204  status = cudnnBackendFinalize(m_heuristics.pointer->get_backend_descriptor());
205  if (status != CUDNN_STATUS_SUCCESS) {
207  &m_heuristics, status, "CUDNN_BACKEND_ENGINEHEUR_DESCRIPTOR: cudnn Finalize failed");
208  return std::move(m_heuristics);
209  };
210 
211  getLogger() << "[cudnn_frontend] " << m_heuristics << std::endl;
212  return std::move(m_heuristics);
213  }
214 
215  explicit EngineHeuristicsBuilder_v8() = default;
216  ~EngineHeuristicsBuilder_v8() = default;
220  operator=(EngineHeuristicsBuilder_v8 const &) = delete;
221 
222  private:
224 };
225 
226 template<std::size_t SIZE>
228 get_heuristics_list(std::array<cudnnBackendHeurMode_t, SIZE> modes,
230  std::function<bool(cudnnBackendDescriptor_t)> filter_fn) {
231  (void) modes;
232  EngineConfigList filtered_configs;
233 
234 
235  for (auto mode : modes) {
236  auto heuristics = EngineHeuristicsBuilder_v8()
237  .setOperationGraph(opGraph)
238  .setHeurMode(mode)
239  .build();
240 
241  auto& engine_config = heuristics.getEngineConfig(heuristics.getEngineConfigCount());
242  cudnn_frontend::filter(engine_config, filtered_configs, filter_fn);
243  }
244 
245  return filtered_configs;
246 }
247 }
auto setHeurMode(cudnnBackendHeurMode_t mode_) -> EngineHeuristicsBuilder_v8 &
Set cudnnHandle for the operations.
ConditionalStreamer & getLogger()
static void set_error_and_throw_exception(BackendDescriptor const *desc, cudnnStatus_t status, const char *message)
auto getEngineConfig(int64_t count=1) -> std::vector< ManagedOpaqueDescriptor > &
Query the total count of the engines for the Operation Set.
static auto filter(Predicate pred, executionPlans_t &plans) -> executionPlans_t
static ManagedOpaqueDescriptor make_shared_backend_pointer(cudnnBackendDescriptorType_t type)
auto setOperationGraph(OperationGraph_v8 &opGraph_) -> EngineHeuristicsBuilder_v8 &
Set operationGraph for the engine (opGraph is not destroyed)
std::string opGraphTag
storage of heuristic results
auto getEngineConfigCount(void) const -> int64_t
Query the total count of the engine config for the Operation Set.
EngineConfigList get_heuristics_list(std::array< cudnnBackendHeurMode_t, SIZE > modes, OperationGraph_v8 &opGraph, std::function< bool(cudnnBackendDescriptor_t)> filter_fn)
j template void())
Definition: json.hpp:4061
std::shared_ptr< OpaqueBackendPointer > ManagedOpaqueDescriptor
EngineHeuristics_v8 & operator=(EngineHeuristics_v8 &&from)=default
std::vector< ManagedOpaqueDescriptor > m_heuristic_results
std::vector< ManagedOpaqueDescriptor > EngineConfigList
std::string describe() const override
Return a string describing the backend Descriptor.
cudnnStatus_t status
Shared pointer of the OpaqueBackendPointer.