Compute Library
 21.11
GEMMBenchmarkResultRecorder Class Reference

Public Member Functions

def __init__ (self, tol=0.01)
 
def add
 
def get_record (self)
 
def get_best_gemm_configs (self)
 
def get_best_gemm_configs_as_sequence (self)
 
def get_config_distributions (self)
 
def get_best_gemm_strategies (self)
 
def save_to_jsons (self, out_dir, only_best_config=True)
 
def summary (self, sum_level=SummaryLevel.Short)
 

Static Public Attributes

 SummaryLevel = Enum("SummaryLevel", ["Short", "Detailed"])
 

Detailed Description

A recorder that records and organises GEMM Benchmark results, and produces various reports on the record.

Definition at line 201 of file GemmTuner.py.

Constructor & Destructor Documentation

◆ __init__()

def __init__ (   self,
  tol = 0.01 
)
Initializer

Definition at line 207 of file GemmTuner.py.

References GEMMBenchmarkResultRecorder._strategies, GEMMBenchmarkResultRecorder._tol, and arm_compute::test::validation.set().

207  def __init__(self, tol=0.01):
208  """ Initializer
209  """
210  self._benchmark_result_record: List[BenchmarkResult] = []
211  # Strategies recorded
212  self._strategies = set()
213  self._tol = tol
214 
dst_shape set(0, output_wh.first)

Member Function Documentation

◆ add()

def add (   self,
  benchmark_result 
)

Definition at line 215 of file GemmTuner.py.

References GEMMBenchmarkResultRecorder._strategies.

Referenced by GEMMBenchmarkResultRecorder.get_config_distributions().

215  def add(self, benchmark_result: BenchmarkResult):
216  """ Add a benchmark result to the record.
217  """
218  gemm_param, strategy, gemm_config, measurement = benchmark_result
219  # Update strategies encoutnered
220  self._strategies.add(strategy)
221 
222  self._benchmark_result_record.append(benchmark_result)
223 

◆ get_best_gemm_configs()

def get_best_gemm_configs (   self)
Get the best GEMMConfig set per GEMMParam per Strategy

Definition at line 229 of file GemmTuner.py.

References GEMMBenchmarkResultRecorder._tol, and GEMMBenchmarkResultRecorder.get_record().

Referenced by GEMMBenchmarkResultRecorder.get_best_gemm_configs_as_sequence(), and GEMMBenchmarkResultRecorder.summary().

229  def get_best_gemm_configs(self):
230  """ Get the best GEMMConfig set per GEMMParam per Strategy
231  """
232  best_gc_sets: Dict[
233  Tuple[GEMMParam, Strategy], List[Tuple[GEMMConfig, Measurement]]
234  ] = defaultdict(list)
235  for gemm_param, strategy, gemm_config, measurement in self.get_record():
236  best_gc_set = best_gc_sets.setdefault((gemm_param, strategy), [])
237  best_gc_set.append((gemm_config, measurement))
238  # Sort the best config set (list)
239  best_gc_set = sorted(
240  best_gc_set, key=lambda gc_and_m: gc_and_m[1].get_total_ms()
241  )
242  # Filter out configs that are beyond tolerance to the best GEMMConfig's measurement
243  best_gc, best_m = best_gc_set[0]
244  best_gc_set_new = [
245  (gemm_config, measurement)
246  for gemm_config, measurement in best_gc_set[1:]
247  if measurement.is_close_to(best_m, self._tol)
248  ]
249  # Add back the best config
250  best_gc_set_new.insert(0, (best_gc, best_m))
251  best_gc_sets[(gemm_param, strategy)] = best_gc_set_new
252 
253  return best_gc_sets
254 

◆ get_best_gemm_configs_as_sequence()

def get_best_gemm_configs_as_sequence (   self)
Get the best GEMMConfig set per GEMMParam per Strategy, and flatten the result into a sequence
of BenchmarkResults

Definition at line 255 of file GemmTuner.py.

References GEMMBenchmarkResultRecorder.get_best_gemm_configs().

Referenced by GEMMBenchmarkResultRecorder.get_config_distributions(), and GEMMBenchmarkResultRecorder.save_to_jsons().

255  def get_best_gemm_configs_as_sequence(self):
256  """ Get the best GEMMConfig set per GEMMParam per Strategy, and flatten the result into a sequence
257  of BenchmarkResults
258  """
259  for (
260  (gemm_param, strategy),
261  best_gc_sets,
262  ) in self.get_best_gemm_configs().items():
263  for best_gemm_config, best_measurement in best_gc_sets:
264  yield BenchmarkResult(
265  gemm_param, strategy, best_gemm_config, best_measurement
266  )
267 

◆ get_best_gemm_strategies()

def get_best_gemm_strategies (   self)
Get the best Stratey per GEMMParam

Definition at line 280 of file GemmTuner.py.

References GEMMBenchmarkResultRecorder.get_record().

Referenced by GEMMBenchmarkResultRecorder.save_to_jsons().

280  def get_best_gemm_strategies(self):
281  """ Get the best Stratey per GEMMParam
282  """
283  all_results: Dict[GEMMParam, List[Tuple[Strategy, Measurement]]] = defaultdict(
284  list
285  )
286 
287  best_strategies: Dict[GEMMParam, Strategy] = {}
288 
289  for gemm_param, strategy, gemm_config, measurement in self.get_record():
290  all_results[gemm_param].append((strategy, measurement))
291 
292  for gemm_param, results_set in all_results.items():
293  # Sort the best results set (list)
294  results_set = sorted(
295  results_set, key=lambda s_and_m: s_and_m[1].get_total_ms()
296  )
297  # Select best Strategy
298  best_s, best_m = results_set[0]
299  best_strategies[gemm_param] = best_s
300 
301  return best_strategies
302 

◆ get_config_distributions()

def get_config_distributions (   self)
Return GEMMConfigDistribution for each strategy

Definition at line 268 of file GemmTuner.py.

References GEMMBenchmarkResultRecorder.add(), and GEMMBenchmarkResultRecorder.get_best_gemm_configs_as_sequence().

268  def get_config_distributions(self):
269  """ Return GEMMConfigDistribution for each strategy
270  """
271  gemm_config_distributions: Dict[Strategy, GEMMConfigDistribution] = defaultdict(
272  GEMMConfigDistribution
273  )
274  for benchmark_result in self.get_best_gemm_configs_as_sequence():
275  _, strategy, _, _ = benchmark_result
276  gemm_config_distributions[strategy].add(benchmark_result)
277 
278  return gemm_config_distributions
279 

◆ get_record()

def get_record (   self,
  Generator,
  BenchmarkResult,
  None,
  None 
)
Return an iterator that iterates over the record.

Definition at line 224 of file GemmTuner.py.

Referenced by GEMMBenchmarkResultRecorder.get_best_gemm_configs(), GEMMBenchmarkResultRecorder.get_best_gemm_strategies(), GEMMBenchmarkResultRecorder.save_to_jsons(), and GEMMBenchmarkResultRecorder.summary().

224  def get_record(self) -> Generator[BenchmarkResult, None, None]:
225  """ Return an iterator that iterates over the record.
226  """
227  yield from self._benchmark_result_record
228 

◆ save_to_jsons()

def save_to_jsons (   self,
  out_dir,
  only_best_config = True 
)
Save records to an output directory of JSON files.
The directory is organized such that each strategy gets its own JSON file.
The directory also includes a JSON file to define the best strategy per GEMM Param.

Definition at line 303 of file GemmTuner.py.

References GEMMBenchmarkResultRecorder._strategies, GemmTuner.check_out_path(), GemmTuner.dump_json(), GEMMBenchmarkResultRecorder.get_best_gemm_configs_as_sequence(), GEMMBenchmarkResultRecorder.get_best_gemm_strategies(), GEMMBenchmarkResultRecorder.get_record(), and caffe_data_extractor.str.

303  def save_to_jsons(self, out_dir, only_best_config=True):
304  """ Save records to an output directory of JSON files.
305  The directory is organized such that each strategy gets its own JSON file.
306  The directory also includes a JSON file to define the best strategy per GEMM Param.
307  """
308  if not os.path.exists(out_dir):
309  logging.info(
310  "Output directory {} does not exist. Creating...".format(
311  out_dir)
312  )
313  os.mkdir(out_dir)
314 
315  out_json_path = os.path.join(out_dir, "gemm_type_selection.json")
316  if check_out_path(out_json_path):
317  results = self.get_best_gemm_strategies()
318  results = {str(key): value.name for key, value in results.items()}
319  dump_json(out_json_path, results)
320 
321  for strategy in self._strategies:
322  out_json_path = os.path.join(
323  out_dir, ("gemm_config_" + strategy.name.lower() + ".json")
324  )
325  if check_out_path(out_json_path):
326  record = (
327  self.get_best_gemm_configs_as_sequence()
328  if only_best_config
329  else self.get_record()
330  )
331  results = defaultdict(list)
332  for res in record:
333  if res.strategy == strategy:
334  results[str(res.gemm_param)].append(
335  {
336  "GEMMConfig": str(res.gemm_config),
337  "OpenCL_Timer_ms_reshape": str(
338  res.measurement.opencl_timer_ms_reshape
339  ),
340  "OpenCL_Timer_ms_kernel": str(
341  res.measurement.opencl_timer_ms_kernel
342  ),
343  }
344  )
345  dump_json(out_json_path, results)
346 
def check_out_path(out_path)
Definition: GemmTuner.py:566
def dump_json(out_path, dict)
Definition: GemmTuner.py:582

◆ summary()

def summary (   self,
  sum_level = SummaryLevel.Short 
)
Return the summary string of the record

Definition at line 347 of file GemmTuner.py.

References GEMMBenchmarkResultRecorder.get_best_gemm_configs(), GEMMBenchmarkResultRecorder.get_record(), and arm_compute::test.join().

347  def summary(self, sum_level=SummaryLevel.Short):
348  """ Return the summary string of the record
349  """
350  num_raw_records = sum(1 for _ in self.get_record())
351  gemm_params_per_strategy = defaultdict(list)
352  for gemm_param, strategy in self.get_best_gemm_configs().keys():
353  gemm_params_per_strategy[strategy].append(gemm_param)
354  global_summary = f"""
355 === {self.__class__.__name__} Summary ===
356 [Global]
357 Strategies recorded: {", ".join(map(lambda s: s.name, self._strategies))}
358 Total number of results recorded: {num_raw_records}
359 
360 [Per strategy]
361  """
362  strategy_summaries = []
363  for strategy in gemm_params_per_strategy:
364  summary = f"""
365 Strategy {strategy.name}:
366 GEMM parameters:
367  Number of: {len(gemm_params_per_strategy[strategy])}
368  """
369  if sum_level == self.__class__.SummaryLevel.Detailed:
370  summary += f"""
371  Content: {gemm_params_per_strategy[strategy]}
372  """
373  strategy_summaries.append(summary)
374  return global_summary + "".join(strategy_summaries)
375 
376 
std::string join(T first, T last, const std::string &separator)
Helper function to concatenate multiple strings.
Definition: Utils.h:93

Field Documentation

◆ SummaryLevel

SummaryLevel = Enum("SummaryLevel", ["Short", "Detailed"])
static

Definition at line 205 of file GemmTuner.py.


The documentation for this class was generated from the following file: