CUDNN Frontend API  8.3.0
cudnn_frontend_ExecutionPlan.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20  * DEALINGS IN THE SOFTWARE.
21  */
22 
23 #pragma once
24 
25 #include <algorithm>
26 #include <array>
27 #include <functional>
28 #include <memory>
29 #include <sstream>
30 #include <utility>
31 
32 #include <cudnn.h>
33 #include <cudnn_backend.h>
34 
35 #include "cudnn_frontend_Engine.h"
36 #include "cudnn_frontend_utils.h"
37 
38 namespace cudnn_frontend {
52  public:
54 
55  ExecutionPlan_v8(ExecutionPlan_v8 &&from) = default;
57  operator=(ExecutionPlan_v8 &&) = default;
58 
59  ~ExecutionPlan_v8() = default;
64  auto
66  getWorkspaceSize(void) const -> int64_t {
67  return workSpaceSize;
68  }
69 
70  std::string
71  describe() const override {
72  std::stringstream ss;
73  ss << "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR : ";
74  ss << getTag() << ", ";
75  ss << "numeric_notes:" << " ";
76  for (auto note : numeric_notes)
77  ss << note << " ";
78 #if (CUDNN_VERSION >= 8200)
79  ss << "behavior_notes:" << " ";
80  for (auto note : behavior_notes)
81  ss << note << " ";
82 #endif
83  ss << "workSpaceSize: " << workSpaceSize;
84  return ss.str();
85  }
86 
87  std::string const &
88  getTag() const {
89  return planTag;
90  }
91 
92  void
93  setExecutionTime(float time_) {
94  execution_time_ms = time_;
95  }
96 
97  float
98  getExecutionTime() const {
99  return execution_time_ms;
100  }
101 
102  std::array<cudnnBackendNumericalNote_t,CUDNN_NUMERICAL_NOTE_TYPE_COUNT> const &
103  getNumericNotes() const {
104  return numeric_notes;
105  }
106 
107 #if (CUDNN_VERSION >= 8200)
108  std::array<cudnnBackendBehaviorNote_t, CUDNN_BEHAVIOR_NOTE_TYPE_COUNT> const &
109  getBehaviorNotes() const {
110  return behavior_notes;
111  }
112 #endif
113 
114  ExecutionPlan_v8(ExecutionPlan_v8 const &) = default;
116  operator=(ExecutionPlan_v8 const &) = default;
117  private:
118  void
120  auto status = CUDNN_STATUS_SUCCESS;
121  int64_t elem_count = 0;
122  cudnnBackendDescriptor_t extractedEngine_ = extractedEngine->get_backend_descriptor();
123  status = cudnnBackendGetAttribute(extractedEngine_,
124  CUDNN_ATTR_ENGINE_NUMERICAL_NOTE,
125  CUDNN_TYPE_NUMERICAL_NOTE,
126  CUDNN_NUMERICAL_NOTE_TYPE_COUNT,
127  &elem_count,
128  numeric_notes.data());
129  if (status != CUDNN_STATUS_SUCCESS) {
131  status,
132  "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: GetAttribute "
133  "CUDNN_ATTR_ENGINE_NUMERICAL_NOTE Failed");
134  }
135 #if (CUDNN_VERSION >= 8200)
136  status = cudnnBackendGetAttribute(extractedEngine_,
137  CUDNN_ATTR_ENGINE_BEHAVIOR_NOTE,
138  CUDNN_TYPE_BEHAVIOR_NOTE,
139  CUDNN_BEHAVIOR_NOTE_TYPE_COUNT,
140  &elem_count,
141  behavior_notes.data());
142  if (status != CUDNN_STATUS_SUCCESS) {
144  status,
145  "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: GetAttribute "
146  "CUDNN_ATTR_ENGINE_BEHAVIOR_NOTE Failed");
147  }
148 #endif
149  }
150 
151  void
152  buildTag(ManagedOpaqueDescriptor &extractedEngine) {
153  // Compute a unique tag for execution plan:
154  auto status = CUDNN_STATUS_SUCCESS;
155  std::stringstream tag{""};
156  int64_t elemCount = 0, engineId = 0, numKnobs = 0;
157 
158  std::array<ManagedOpaqueDescriptor, CUDNN_KNOB_TYPE_COUNTS> extractedKnobs{{nullptr}};
159  for (auto &knob : extractedKnobs) {
160  knob = make_shared_backend_pointer(CUDNN_BACKEND_KNOB_CHOICE_DESCRIPTOR);
161  status = knob->get_status();
162  if (status != CUDNN_STATUS_SUCCESS) {
164  this, status, "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: cudnnCreate Failed when compute tag");
165  }
166  }
167 
168  cudnnBackendDescriptor_t extractedEngine_ = extractedEngine->get_backend_descriptor();
169  std::array<cudnnBackendDescriptor_t, CUDNN_KNOB_TYPE_COUNTS> extractedKnobs_{{nullptr}};
170  for (std::uint32_t i = 0; i < extractedKnobs.size(); i++) {
171  extractedKnobs_[i] = extractedKnobs[i]->get_backend_descriptor();
172  }
173 
174  status = cudnnBackendGetAttribute(
175  extractedEngine_, CUDNN_ATTR_ENGINE_GLOBAL_INDEX, CUDNN_TYPE_INT64, 1, &elemCount, &engineId);
176  if (status != CUDNN_STATUS_SUCCESS) {
178  status,
179  "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: GetAttribute "
180  "CUDNN_ATTR_ENGINE_GLOBAL_INDEX Failed");
181  }
182  tag << "eng" << engineId;
183 
184  status = cudnnBackendGetAttribute(engine_config->get_backend_descriptor(),
185  CUDNN_ATTR_ENGINECFG_KNOB_CHOICES,
186  CUDNN_TYPE_BACKEND_DESCRIPTOR,
187  CUDNN_KNOB_TYPE_COUNTS,
188  &numKnobs,
189  &(extractedKnobs_[0]));
190  if (status != CUDNN_STATUS_SUCCESS) {
192  status,
193  "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: GetAttribute "
194  "CUDNN_ATTR_ENGINECFG_KNOB_CHOICES Failed");
195  }
196  if (numKnobs > CUDNN_KNOB_TYPE_COUNTS) {
198  status,
199  "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: GetAttribute "
200  "numKnobs exceed the CUDNN_KNOB_TYPE_COUNTS");
201  }
202  for (int64_t idx = 0; idx < numKnobs; ++idx) {
203  const cudnnBackendDescriptor_t &knob = extractedKnobs_[idx];
204  cudnnBackendKnobType_t type = CUDNN_KNOB_TYPE_COUNTS;
205  int64_t choice = -2;
206  status = cudnnBackendGetAttribute(
207  knob, CUDNN_ATTR_KNOB_CHOICE_KNOB_TYPE, CUDNN_TYPE_KNOB_TYPE, 1, nullptr, &type);
208  if (status != CUDNN_STATUS_SUCCESS) {
210  status,
211  "computeTag CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: GetAttribute "
212  "CUDNN_ATTR_KNOB_CHOICE_KNOB_TYPE Failed");
213  }
214  status = cudnnBackendGetAttribute(
215  knob, CUDNN_ATTR_KNOB_CHOICE_KNOB_VALUE, CUDNN_TYPE_INT64, 1, nullptr, &choice);
216  if (status != CUDNN_STATUS_SUCCESS) {
218  status,
219  "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: GetAttribute "
220  "CUDNN_ATTR_KNOB_CHOICE_KNOB_VALUE Failed");
221  }
222  tag << "_k" << type << "=" << choice;
223  }
224  planTag += tag.str();
225 
226  }
227 
228  void
230  auto status = cudnnBackendGetAttribute(pointer->get_backend_descriptor(),
231  CUDNN_ATTR_EXECUTION_PLAN_WORKSPACE_SIZE,
232  CUDNN_TYPE_INT64,
233  1,
234  NULL,
235  &workSpaceSize);
236  if (status != CUDNN_STATUS_SUCCESS) {
238  status,
239  "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: GetAttribute "
240  "CUDNN_ATTR_EXECUTION_PLAN_WORKSPACE_SIZE Failed");
241  }
242  if (workSpaceSize < 0) {
244  this, status, "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: GetAttribute Workspace Size Invalid");
245  }
246  }
247 
248  ExecutionPlan_v8() = default;
250  cudnnHandle_t handle = nullptr;
251  std::string planTag;
252 
253  std::int64_t workSpaceSize = 0;
254  std::array<cudnnBackendNumericalNote_t,CUDNN_NUMERICAL_NOTE_TYPE_COUNT> numeric_notes;
255 #if (CUDNN_VERSION >= 8200)
256  std::array<cudnnBackendBehaviorNote_t, CUDNN_BEHAVIOR_NOTE_TYPE_COUNT> behavior_notes;
257 #endif
258 
259  float execution_time_ms = 0.0f;
260 
261 };
262 
267  public:
272  auto
274  setHandle(cudnnHandle_t handle_) -> ExecutionPlanBuilder_v8 & {
275  m_execution_plan.handle = handle_;
276  return *this;
277  }
279  auto
281  m_execution_plan.engine_config = engine_config_.get_desc();
282  m_execution_plan.planTag = engine_config_.getTag();
283  return *this;
284  }
285 
287  auto
288  setEngineConfig(ManagedOpaqueDescriptor &desc, std::string const &opGraphTag_ = "") -> ExecutionPlanBuilder_v8 & {
289  m_execution_plan.engine_config = desc;
290  m_execution_plan.planTag = opGraphTag_;
291  return *this;
292  }
298  build() {
299  if (m_execution_plan.handle == nullptr) {
301  &m_execution_plan,
302  CUDNN_STATUS_BAD_PARAM,
303  "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: Check and Set the CUDNN_ATTR_EXECUTION_PLAN_HANDLE");
304  return std::move(m_execution_plan);
305  };
306  if (m_execution_plan.engine_config == nullptr) {
308  &m_execution_plan,
309  CUDNN_STATUS_BAD_PARAM,
310  "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: Check and Set the CUDNN_ATTR_EXECUTION_PLAN_ENGINE_CONFIG");
311  return std::move(m_execution_plan);
312  };
313 
314  // Create a descriptor. Memory allocation happens here.
315  auto status = m_execution_plan.initialize_managed_backend_pointer(CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR);
316  if (status != CUDNN_STATUS_SUCCESS) {
318  &m_execution_plan, status, "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: cudnnCreate Failed");
319  return std::move(m_execution_plan);
320  }
321 
322  status = cudnnBackendSetAttribute(m_execution_plan.pointer->get_backend_descriptor(),
323  CUDNN_ATTR_EXECUTION_PLAN_ENGINE_CONFIG,
324  CUDNN_TYPE_BACKEND_DESCRIPTOR,
325  1,
326  &(m_execution_plan.engine_config->get_backend_descriptor()));
327  if (status != CUDNN_STATUS_SUCCESS) {
329  &m_execution_plan,
330  status,
331  "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: SetAttribute CUDNN_ATTR_EXECUTION_PLAN_ENGINE_CONFIG Failed");
332  return std::move(m_execution_plan);
333  }
334  status = cudnnBackendSetAttribute(m_execution_plan.pointer->get_backend_descriptor(),
335  CUDNN_ATTR_EXECUTION_PLAN_HANDLE,
336  CUDNN_TYPE_HANDLE,
337  1,
338  &m_execution_plan.handle);
339  if (status != CUDNN_STATUS_SUCCESS) {
341  &m_execution_plan,
342  status,
343  "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: SetAttribute CUDNN_ATTR_EXECUTION_PLAN_HANDLE Failed");
344  return std::move(m_execution_plan);
345  }
346  // Finalizing the descriptor
347  status = cudnnBackendFinalize(m_execution_plan.pointer->get_backend_descriptor());
348  if (status != CUDNN_STATUS_SUCCESS) {
350  &m_execution_plan, status, "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: cudnnFinalize Descriptor Failed");
351  return std::move(m_execution_plan);
352  }
353 
354  ManagedOpaqueDescriptor extractedEngine = make_shared_backend_pointer(CUDNN_BACKEND_ENGINE_DESCRIPTOR);
355  status = extractedEngine->get_status();
356  if (status != CUDNN_STATUS_SUCCESS) {
358  &m_execution_plan, status, "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: cudnnCreate of CUDNN_BACKEND_ENGINE_DESCRIPTOR failed when compute tag");
359  return std::move(m_execution_plan);
360  }
361  cudnnBackendDescriptor_t extractedEngine_ = extractedEngine->get_backend_descriptor();
362  int64_t elemCount = 0;
363  status = cudnnBackendGetAttribute(m_execution_plan.engine_config->get_backend_descriptor(),
364  CUDNN_ATTR_ENGINECFG_ENGINE,
365  CUDNN_TYPE_BACKEND_DESCRIPTOR,
366  1,
367  &elemCount,
368  &extractedEngine_);
369  if (status != CUDNN_STATUS_SUCCESS) {
370  set_error_and_throw_exception(&m_execution_plan,
371  status,
372  "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: GetAttribute "
373  "CUDNN_ATTR_ENGINECFG_ENGINE Failed");
374  return std::move(m_execution_plan);
375  }
376 
377  m_execution_plan.buildTag(extractedEngine);
378  m_execution_plan.fetchNotes(extractedEngine);
379  m_execution_plan.computeWorkSpaceSize();
380 
381  getLogger() << "[cudnn_frontend] " << m_execution_plan << std::endl;
382  return std::move(m_execution_plan);
383  }
384 
385  explicit ExecutionPlanBuilder_v8() = default;
386  ~ExecutionPlanBuilder_v8() = default;
390  operator=(ExecutionPlanBuilder_v8 const &) = delete;
391 
392  private:
394 };
395 
398 }
auto setEngineConfig(EngineConfig_v8 const &engine_config_) -> ExecutionPlanBuilder_v8 &
Set engine Config for the Plan.
ConditionalStreamer & getLogger()
static void set_error_and_throw_exception(BackendDescriptor const *desc, cudnnStatus_t status, const char *message)
static ManagedOpaqueDescriptor make_shared_backend_pointer(cudnnBackendDescriptorType_t type)
void buildTag(ManagedOpaqueDescriptor &extractedEngine)
auto setEngineConfig(ManagedOpaqueDescriptor &desc, std::string const &opGraphTag_="") -> ExecutionPlanBuilder_v8 &
Set engine Config for the Plan.
void fetchNotes(ManagedOpaqueDescriptor &extractedEngine)
auto setHandle(cudnnHandle_t handle_) -> ExecutionPlanBuilder_v8 &
Set engine for the ExecutionPlan_v8.
std::shared_ptr< OpaqueBackendPointer > ManagedOpaqueDescriptor
std::string describe() const override
Return a string describing the backend Descriptor.
ExecutionPlan_v8 & operator=(ExecutionPlan_v8 &&)=default
std::array< cudnnBackendNumericalNote_t, CUDNN_NUMERICAL_NOTE_TYPE_COUNT > const & getNumericNotes() const
std::array< cudnnBackendNumericalNote_t, CUDNN_NUMERICAL_NOTE_TYPE_COUNT > numeric_notes
cudnnStatus_t status
Shared pointer of the OpaqueBackendPointer.
auto getWorkspaceSize(void) const -> int64_t
Query the workspace requirement for the given plan.