CUDNN Frontend API  8.3.0
cudnn_frontend_ExecutionPlanCache.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20  * DEALINGS IN THE SOFTWARE.
21  */
22 
23 #include <tuple>
24 #include <unordered_map>
25 #include <map>
26 #include <memory>
27 #include <mutex>
28 
31 
37 
40 /***
41  * device_id_0 Operation_Graph0 (conv_fprop)
42  * -------------------------------------------------------------------------------
43  * | Feature_vector_type0_val0 | Plan A0 |
44  * | Feature_vector_type0_val1 | Plan B0 |
45  * ===============================================================================
46  *
47  * device_id_0 Operation_Graph1 (dgrad)
48  * -------------------------------------------------------------------------------
49  * | Feature_vector_type1_val0 | Plan A1 |
50  * | Feature_vector_type1_val1 | Plan B1 |
51  * ===============================================================================
52  *
53  * device_id_0 Operation_Graph2 (wgrad)
54  * -------------------------------------------------------------------------------
55  * | Feature_vector_type2_val0 | Plan B2 |
56  * ===============================================================================
57  *
58  * device_id_1 Operation_Graph0 (conv_fprop)
59  * -------------------------------------------------------------------------------
60  * | Feature_vector_type0_val0 | Plan A0 |
61  * | Feature_vector_type0_val1 | Plan B0 |
62  * ===============================================================================
63  *
64  * device_id_1 Operation_Graph1 (dgrad)
65  * -------------------------------------------------------------------------------
66  * | Feature_vector_type1_val0 | Plan A1 |
67  * | Feature_vector_type1_val1 | Plan B1 |
68  * ===============================================================================
69  *
70  * device_id_1 Operation_Graph2 (wgrad)
71  * -------------------------------------------------------------------------------
72  * | Feature_vector_type2_val0 | Plan B2 |
73  * ===============================================================================
74  */
75 
76 namespace cudnn_frontend {
77 
80 
81  protected:
82  struct compare {
83  bool operator ()(const feature_vector_t & fv1, const feature_vector_t &fv2) const {
84  for (uint32_t i = 0u; i < fv1.size(); i++) {
85  if (fv1[i] < fv2[i]) {
86  return true;
87  }
88  }
89  }
90  };
91 
92  std::string name = "plan_cache_[unnamed]";
93 
96  using FeatureVectorToPlanMap = std::map<cudnn_frontend::feature_vector_t, cudnn_frontend::ExecutionPlan, cudnn_frontend::ExecutionPlanCache_v1::compare>;
98 
99  mutable std::mutex cache_mutex;
100 
101  public:
102  virtual bool
104  const std::string & tag) {
105  return true;
106  }
107 
109  const cudnn_frontend::ExecutionPlan &plan) {
110  std::lock_guard<std::mutex> guard(cache_mutex);
111  cache.insert(std::make_pair(op_graph.getFeatureVector(),plan));
112  getLogger() << "[cudnn_frontend] Added to " << name << " " << op_graph.getTag() << std::endl;
113  }
114 
115  ExecutionPlanCache_v1(const char * name_) {
116  name = name_;
117  }
118 
119  const std::string & get_name() const {
120  return name;
121  }
122 
123  // Plan is the output here.
125  const cudnn_frontend::ExecutionPlan *&plan) const {
126  {
127  std::lock_guard<std::mutex> guard(cache_mutex);
128  auto it = cache.find(op_graph.getFeatureVector());
129 
130  if (it == cache.end()) {
131  getLogger() << "[cudnn_frontend] Cached Plan Not Found in " << name << std::endl;
132  return false;
133  }
134  plan = &(it->second);
135  }
136  getLogger() << "[cudnn_frontend] Cached Plan Found in " << name << std::endl;
137  return true;
138  }
139 };
140 
142 
143  using SaturationTracker = std::map<std::pair<cudnn_frontend::feature_vector_t, std::string>, int32_t>;
145 
146  int32_t saturationCount = 1;
147 
148  public:
149  virtual bool
151  const std::string & tag) {
152  if (saturationCount == 1) {return true;} // Special case. Always add to the cache.
153 
154  // If plan cache is already created for the op_graph no need to update.
155  // Ideally, one will auto-tune only if the plan cache has no plan for the op_graph.
156  cudnn_frontend::ExecutionPlan const *plan = nullptr;
157  if (get_plan_from_cache(op_graph, plan)) {
158  getLogger() << "[cudnn_frontend] SaturationTracker " << name << " " << op_graph.getTag() << " " << tag << " plan already in cache." << std::endl;
159  return false;
160  }
161 
162  // Lock the cache and increase the count till we saturate
163  std::lock_guard<std::mutex> guard(cache_mutex);
164  auto cnt = tracker[std::make_pair(op_graph.getFeatureVector(),tag)] += 1;
165  getLogger() << "[cudnn_frontend] SaturationTracker " << name << " " << op_graph.getTag() << " " << tag << " " << cnt << std::endl;
166  return cnt >= saturationCount;
167  }
168 
169  void set_saturation_count(int32_t count) {
170  saturationCount = count;
171  }
172 
173  ExecutionPlanCache_v2(const char * name_) : ExecutionPlanCache_v1(name_) {
174  }
175 
176 };
177 
179 
180 }
std::vector< int64_t > feature_vector_t
Detailed feature_vector. Generally the Tensor and Operation properties.
ConditionalStreamer & getLogger()
Plan Cache structure for the above table.
virtual bool is_fastest_plan_stable(const cudnn_frontend::OperationGraph &op_graph, const std::string &tag)
bool operator()(const feature_vector_t &fv1, const feature_vector_t &fv2) const
std::map< std::pair< cudnn_frontend::feature_vector_t, std::string >, int32_t > SaturationTracker
void add_plan_to_cache(const cudnn_frontend::OperationGraph &op_graph, const cudnn_frontend::ExecutionPlan &plan)
virtual bool is_fastest_plan_stable(const cudnn_frontend::OperationGraph &op_graph, const std::string &tag)
bool get_plan_from_cache(const cudnn_frontend::OperationGraph &op_graph, const cudnn_frontend::ExecutionPlan *&plan) const
std::map< cudnn_frontend::feature_vector_t, cudnn_frontend::ExecutionPlan, cudnn_frontend::ExecutionPlanCache_v1::compare > FeatureVectorToPlanMap