Compute Library
 21.11
Framework.cpp
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2017-2021 Arm Limited.
3  *
4  * SPDX-License-Identifier: MIT
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a copy
7  * of this software and associated documentation files (the "Software"), to
8  * deal in the Software without restriction, including without limitation the
9  * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10  * sell copies of the Software, and to permit persons to whom the Software is
11  * furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in all
14  * copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22  * SOFTWARE.
23  */
24 #include "Framework.h"
25 
29 
30 #ifdef ARM_COMPUTE_CL
33 
34 #endif /* ARM_COMPUTE_CL */
35 
36 #include <chrono>
37 #include <iostream>
38 #include <memory>
39 #include <sstream>
40 #include <type_traits>
41 
42 namespace arm_compute
43 {
44 namespace test
45 {
46 std::unique_ptr<ParametersLibrary> parameters;
47 
48 namespace framework
49 {
50 std::unique_ptr<InstrumentsInfo> instruments_info;
51 
52 Framework::Framework()
53  : _test_filter(nullptr)
54 {
55  _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::WALL_CLOCK_TIMESTAMPS, ScaleFactor::NONE), Instrument::make_instrument<WallClockTimestamps, ScaleFactor::NONE>);
56  _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::WALL_CLOCK_TIMESTAMPS, ScaleFactor::TIME_MS),
57  Instrument::make_instrument<WallClockTimestamps, ScaleFactor::TIME_MS>);
58  _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::WALL_CLOCK_TIMESTAMPS, ScaleFactor::TIME_S),
59  Instrument::make_instrument<WallClockTimestamps, ScaleFactor::TIME_S>);
60  _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::WALL_CLOCK_TIMER, ScaleFactor::NONE), Instrument::make_instrument<WallClockTimer, ScaleFactor::NONE>);
61  _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::WALL_CLOCK_TIMER, ScaleFactor::TIME_MS), Instrument::make_instrument<WallClockTimer, ScaleFactor::TIME_MS>);
62  _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::WALL_CLOCK_TIMER, ScaleFactor::TIME_S), Instrument::make_instrument<WallClockTimer, ScaleFactor::TIME_S>);
63  _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::SCHEDULER_TIMESTAMPS, ScaleFactor::NONE), Instrument::make_instrument<SchedulerTimestamps, ScaleFactor::NONE>);
64  _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::SCHEDULER_TIMESTAMPS, ScaleFactor::TIME_MS),
65  Instrument::make_instrument<SchedulerTimestamps, ScaleFactor::TIME_MS>);
66  _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::SCHEDULER_TIMESTAMPS, ScaleFactor::TIME_S),
67  Instrument::make_instrument<SchedulerTimestamps, ScaleFactor::TIME_S>);
68  _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::SCHEDULER_TIMER, ScaleFactor::NONE), Instrument::make_instrument<SchedulerTimer, ScaleFactor::NONE>);
69  _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::SCHEDULER_TIMER, ScaleFactor::TIME_MS), Instrument::make_instrument<SchedulerTimer, ScaleFactor::TIME_MS>);
70  _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::SCHEDULER_TIMER, ScaleFactor::TIME_S), Instrument::make_instrument<SchedulerTimer, ScaleFactor::TIME_S>);
71 #ifdef PMU_ENABLED
72  _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::PMU, ScaleFactor::NONE), Instrument::make_instrument<PMUCounter, ScaleFactor::NONE>);
73  _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::PMU, ScaleFactor::SCALE_1K), Instrument::make_instrument<PMUCounter, ScaleFactor::SCALE_1K>);
74  _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::PMU, ScaleFactor::SCALE_1M), Instrument::make_instrument<PMUCounter, ScaleFactor::SCALE_1M>);
75 #endif /* PMU_ENABLED */
76 #ifdef MALI_ENABLED
77  _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::MALI, ScaleFactor::NONE), Instrument::make_instrument<MaliCounter, ScaleFactor::NONE>);
78  _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::MALI, ScaleFactor::SCALE_1K), Instrument::make_instrument<MaliCounter, ScaleFactor::SCALE_1K>);
79  _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::MALI, ScaleFactor::SCALE_1M), Instrument::make_instrument<MaliCounter, ScaleFactor::SCALE_1M>);
80 #endif /* MALI_ENABLED */
81 #ifdef ARM_COMPUTE_CL
82  _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::OPENCL_TIMESTAMPS, ScaleFactor::NONE), Instrument::make_instrument<OpenCLTimestamps, ScaleFactor::NONE>);
83  _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::OPENCL_TIMESTAMPS, ScaleFactor::TIME_US), Instrument::make_instrument<OpenCLTimestamps, ScaleFactor::TIME_US>);
84  _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::OPENCL_TIMESTAMPS, ScaleFactor::TIME_MS), Instrument::make_instrument<OpenCLTimestamps, ScaleFactor::TIME_MS>);
85  _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::OPENCL_TIMESTAMPS, ScaleFactor::TIME_S), Instrument::make_instrument<OpenCLTimestamps, ScaleFactor::TIME_S>);
86  _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::OPENCL_TIMER, ScaleFactor::NONE), Instrument::make_instrument<OpenCLTimer, ScaleFactor::NONE>);
87  _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::OPENCL_TIMER, ScaleFactor::TIME_US), Instrument::make_instrument<OpenCLTimer, ScaleFactor::TIME_US>);
88  _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::OPENCL_TIMER, ScaleFactor::TIME_MS), Instrument::make_instrument<OpenCLTimer, ScaleFactor::TIME_MS>);
89  _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::OPENCL_TIMER, ScaleFactor::TIME_S), Instrument::make_instrument<OpenCLTimer, ScaleFactor::TIME_S>);
90  _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::OPENCL_MEMORY_USAGE, ScaleFactor::NONE), Instrument::make_instrument<OpenCLMemoryUsage, ScaleFactor::NONE>);
91  _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::OPENCL_MEMORY_USAGE, ScaleFactor::SCALE_1K),
92  Instrument::make_instrument<OpenCLMemoryUsage, ScaleFactor::SCALE_1K>);
93  _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::OPENCL_MEMORY_USAGE, ScaleFactor::SCALE_1M),
94  Instrument::make_instrument<OpenCLMemoryUsage, ScaleFactor::SCALE_1M>);
95 #endif /* ARM_COMPUTE_CL */
96 
97  instruments_info = std::make_unique<InstrumentsInfo>();
98 }
99 
100 std::set<InstrumentsDescription> Framework::available_instruments() const
101 {
102  std::set<InstrumentsDescription> types;
103 
104  for(const auto &instrument : _available_instruments)
105  {
106  types.emplace(instrument.first);
107  }
108 
109  return types;
110 }
111 
112 std::map<TestResult::Status, int> Framework::count_test_results() const
113 {
114  std::map<TestResult::Status, int> counts;
115 
116  for(const auto &test : _test_results)
117  {
118  ++counts[test.second.status];
119  }
120 
121  return counts;
122 }
123 
125 {
126  static Framework instance;
127  return instance;
128 }
129 
130 void Framework::init(const FrameworkConfig &config)
131 {
132  _test_filter.reset(new TestFilter(config.mode, config.name_filter, config.id_filter));
133  _num_iterations = config.num_iterations;
134  _log_level = config.log_level;
135  _cooldown_sec = config.cooldown_sec;
136  _configure_only = config.configure_only;
137 
138  _instruments = std::set<framework::InstrumentsDescription>(std::begin(config.instruments), std::end(config.instruments));
139 }
140 
141 std::string Framework::current_suite_name() const
142 {
143  return join(_test_suite_name.cbegin(), _test_suite_name.cend(), "/");
144 }
145 
146 void Framework::push_suite(std::string name)
147 {
148  _test_suite_name.emplace_back(std::move(name));
149 }
150 
152 {
153  _test_suite_name.pop_back();
154 }
155 
157 {
158  _test_info.emplace_back(std::move(info));
159 }
160 
162 {
163  _test_info.clear();
164 }
165 
167 {
168  return !_test_info.empty();
169 }
170 
171 void Framework::print_test_info(std::ostream &os) const
172 {
173  if(!_test_info.empty())
174  {
175  os << "CONTEXT:\n";
176 
177  for(const auto &str : _test_info)
178  {
179  os << " " << str << "\n";
180  }
181  }
182 }
183 
184 template <typename F>
185 void Framework::func_on_all_printers(F &&func)
186 {
187  std::for_each(std::begin(_printers), std::end(_printers), func);
188 }
189 
191 {
192  if(_log_level >= LogLevel::TESTS)
193  {
194  func_on_all_printers([&](Printer * p)
195  {
196  p->print_test_header(info);
197  });
198  }
199 }
200 
202 {
203  static_cast<void>(info);
204 }
205 
207 {
208  if(_log_level >= LogLevel::MEASUREMENTS)
209  {
210  func_on_all_printers([&](Printer * p)
211  {
212  p->print_profiler_header(_test_results.at(info).header_data);
213  p->print_measurements(_test_results.at(info).measurements);
214  });
215  }
216 
217  if(_log_level >= LogLevel::TESTS)
218  {
219  func_on_all_printers([](Printer * p)
220  {
221  p->print_test_footer();
222  });
223  }
224 }
225 
227 {
228  ARM_COMPUTE_ERROR_ON(_current_test_info == nullptr);
229  ARM_COMPUTE_ERROR_ON(_current_test_result == nullptr);
230 
231  const bool is_expected_failure = _current_test_info->status == TestCaseFactory::Status::EXPECTED_FAILURE;
232 
233  if(_log_level >= error.level())
234  {
235  func_on_all_printers([&](Printer * p)
236  {
237  p->print_error(error, is_expected_failure);
238  });
239  }
240 
241  _current_test_result->status = TestResult::Status::FAILED;
242 }
243 
244 void Framework::log_info(const std::string &info)
245 {
246  if(_log_level >= LogLevel::DEBUG)
247  {
248  func_on_all_printers([&](Printer * p)
249  {
250  p->print_info(info);
251  });
252  }
253 }
254 
256 {
257  return _num_iterations;
258 }
259 
261 {
262  _num_iterations = num_iterations;
263 }
264 
266 {
267  _throw_errors = throw_errors;
268 }
269 
271 {
272  return _throw_errors;
273 }
274 
276 {
277  _stop_on_error = stop_on_error;
278 }
279 
281 {
282  return _stop_on_error;
283 }
284 
286 {
287  _error_on_missing_assets = error_on_missing_assets;
288 }
289 
291 {
292  return _error_on_missing_assets;
293 }
294 
295 void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory)
296 {
297  if(test_factory.status() == TestCaseFactory::Status::DISABLED)
298  {
299  log_test_skipped(info);
301  return;
302  }
303 
304  log_test_start(info);
305 
306  Profiler profiler = get_profiler();
308 
309  _current_test_info = &info;
310  _current_test_result = &result;
311 
312  if(_log_level >= LogLevel::ERRORS)
313  {
314  func_on_all_printers([](Printer * p)
315  {
316  p->print_errors_header();
317  });
318  }
319 
320  const bool is_expected_failure = info.status == TestCaseFactory::Status::EXPECTED_FAILURE;
321 
322  try
323  {
324  std::unique_ptr<TestCase> test_case = test_factory.make();
325 
326  try
327  {
328  profiler.test_start();
329 
330  test_case->do_setup();
331 
332  for(int i = 0; i < _num_iterations; ++i)
333  {
334  //Start the profiler if:
335  //- there is only one iteration
336  //- it's not the first iteration of a multi-iterations run.
337  //
338  //Reason: if the CLTuner is enabled then the first run will be really messy
339  //as each kernel will be executed several times, messing up the instruments like OpenCL timers.
340  if(_num_iterations == 1 || i != 0)
341  {
342  profiler.start();
343  }
344  test_case->do_run();
345  test_case->do_sync();
346  if(_num_iterations == 1 || i != 0)
347  {
348  profiler.stop();
349  }
350  }
351 
352  test_case->do_teardown();
353 
354  profiler.test_stop();
355 
356  // Change status to success if no error has happend
357  if(result.status == TestResult::Status::NOT_RUN)
358  {
360  }
361  }
362  catch(const FileNotFound &error)
363  {
364  profiler.test_stop();
365  if(_error_on_missing_assets)
366  {
367  if(_log_level >= LogLevel::ERRORS)
368  {
369  TestError test_error(error.what(), LogLevel::ERRORS);
370  func_on_all_printers([&](Printer * p)
371  {
372  p->print_error(test_error, is_expected_failure);
373  });
374  }
375 
377 
378  if(_throw_errors)
379  {
380  throw;
381  }
382  }
383  else
384  {
385  if(_log_level >= LogLevel::DEBUG)
386  {
387  func_on_all_printers([&](Printer * p)
388  {
389  p->print_info(error.what());
390  });
391  }
392 
394  }
395  }
396  catch(const TestError &error)
397  {
398  profiler.test_stop();
399  if(_log_level >= error.level())
400  {
401  func_on_all_printers([&](Printer * p)
402  {
403  p->print_error(error, is_expected_failure);
404  });
405  }
406 
408 
409  if(_throw_errors)
410  {
411  throw;
412  }
413  }
414 #ifdef ARM_COMPUTE_CL
415  catch(const ::cl::Error &error)
416  {
417  profiler.test_stop();
418  if(_log_level >= LogLevel::ERRORS)
419  {
420  std::stringstream stream;
421  stream << "Error code: " << error.err();
422  TestError test_error(error.what(), LogLevel::ERRORS, stream.str());
423  func_on_all_printers([&](Printer * p)
424  {
425  p->print_error(test_error, is_expected_failure);
426  });
427  }
428 
430 
431  if(_throw_errors)
432  {
433  throw;
434  }
435  }
436 #endif /* ARM_COMPUTE_CL */
437  catch(const std::exception &error)
438  {
439  profiler.test_stop();
440  if(_log_level >= LogLevel::ERRORS)
441  {
442  func_on_all_printers([&](Printer * p)
443  {
444  p->print_error(error, is_expected_failure);
445  });
446  }
447 
449 
450  if(_throw_errors)
451  {
452  throw;
453  }
454  }
455  catch(...)
456  {
457  profiler.test_stop();
458  if(_log_level >= LogLevel::ERRORS)
459  {
460  func_on_all_printers([&](Printer * p)
461  {
462  p->print_error(TestError("Received unknown exception"), is_expected_failure);
463  });
464  }
465 
467 
468  if(_throw_errors)
469  {
470  throw;
471  }
472  }
473  }
474  catch(const std::exception &error)
475  {
476  if(_log_level >= LogLevel::ERRORS)
477  {
478  func_on_all_printers([&](Printer * p)
479  {
480  p->print_error(error, is_expected_failure);
481  });
482  }
483 
485 
486  if(_throw_errors)
487  {
488  throw;
489  }
490  }
491  catch(...)
492  {
493  if(_log_level >= LogLevel::ERRORS)
494  {
495  func_on_all_printers([&](Printer * p)
496  {
497  p->print_error(TestError("Received unknown exception"), is_expected_failure);
498  });
499  }
500 
502 
503  if(_throw_errors)
504  {
505  throw;
506  }
507  }
508 
509  if(_log_level >= LogLevel::ERRORS)
510  {
511  func_on_all_printers([](Printer * p)
512  {
513  p->print_errors_footer();
514  });
515  }
516 
517  _current_test_info = nullptr;
518  _current_test_result = nullptr;
519 
520  if(result.status == TestResult::Status::FAILED)
521  {
523  {
525  }
526  }
527 
529  {
530  if(_stop_on_error)
531  {
532  throw std::runtime_error("Abort on first error.");
533  }
534  }
535 
536  result.header_data = profiler.header();
537  result.measurements = profiler.measurements();
538 
539  set_test_result(info, result);
540  log_test_end(info);
541 }
542 
544 {
545  // Clear old test results
546  _test_results.clear();
547 
548  if(_log_level >= LogLevel::TESTS)
549  {
550  func_on_all_printers([](Printer * p)
551  {
552  p->print_run_header();
553  });
554  }
555 
556  const std::chrono::time_point<std::chrono::high_resolution_clock> start = std::chrono::high_resolution_clock::now();
557 
558  int id = 0;
559  int id_run_test = 0;
560 
561  for(auto &test_factory : _test_factories)
562  {
563  const std::string test_case_name = test_factory->name();
564  const TestInfo test_info{ id, test_case_name, test_factory->mode(), test_factory->status() };
565 
566  if(_test_filter->is_selected(test_info))
567  {
568 #ifdef ARM_COMPUTE_CL
569  // Every 100 tests, reset the OpenCL context to release the allocated memory
570  if(opencl_is_available() && (id_run_test % 100) == 0)
571  {
572  auto ctx_properties = CLScheduler::get().context().getInfo<CL_CONTEXT_PROPERTIES>(nullptr);
573  auto queue_properties = CLScheduler::get().queue().getInfo<CL_QUEUE_PROPERTIES>(nullptr);
574 
575  cl::Context new_ctx = cl::Context(CL_DEVICE_TYPE_DEFAULT, ctx_properties.data());
576  cl::CommandQueue new_queue = cl::CommandQueue(new_ctx, CLKernelLibrary::get().get_device(), queue_properties);
577 
579  CLScheduler::get().set_context(new_ctx);
580  CLScheduler::get().set_queue(new_queue);
581  }
582 #endif // ARM_COMPUTE_CL
583 
584  run_test(test_info, *test_factory);
585 
586  ++id_run_test;
587 
588  // Run test delay
589  sleep_in_seconds(_cooldown_sec);
590  }
591 
592  ++id;
593  }
594 
595  const std::chrono::time_point<std::chrono::high_resolution_clock> end = std::chrono::high_resolution_clock::now();
596 
597  if(_log_level >= LogLevel::TESTS)
598  {
599  func_on_all_printers([](Printer * p)
600  {
601  p->print_run_footer();
602  });
603  }
604 
605  auto runtime = std::chrono::duration_cast<std::chrono::seconds>(end - start);
606  std::map<TestResult::Status, int> results = count_test_results();
607 
608  if(_log_level > LogLevel::NONE)
609  {
610  std::cout << "Executed " << _test_results.size() << " test(s) ("
611  << results[TestResult::Status::SUCCESS] << " passed, "
612  << results[TestResult::Status::EXPECTED_FAILURE] << " expected failures, "
613  << results[TestResult::Status::FAILED] << " failed, "
614  << results[TestResult::Status::CRASHED] << " crashed, "
615  << results[TestResult::Status::DISABLED] << " disabled) in " << runtime.count() << " second(s)\n";
616  }
617 
618  int num_successful_tests = results[TestResult::Status::SUCCESS] + results[TestResult::Status::EXPECTED_FAILURE] + results[TestResult::Status::DISABLED];
619 
620  return (static_cast<unsigned int>(num_successful_tests) == _test_results.size());
621 }
622 
624 {
625  _test_results.emplace(std::move(info), std::move(result));
626 }
627 
629 {
630  printer.print_run_header();
631 
632  for(const auto &test : _test_results)
633  {
634  printer.print_test_header(test.first);
635  printer.print_profiler_header(test.second.header_data);
636  printer.print_measurements(test.second.measurements);
637  printer.print_test_footer();
638  }
639 
640  printer.print_run_footer();
641 }
642 
644 {
645  Profiler profiler;
646 
647  const bool all_instruments = std::any_of(
648  _instruments.begin(),
649  _instruments.end(),
650  [](InstrumentsDescription type) -> bool { return type.first == InstrumentType::ALL; });
651 
652  auto is_selected = [&](InstrumentsDescription instrument) -> bool
653  {
654  return std::find_if(_instruments.begin(), _instruments.end(), [&](InstrumentsDescription type) -> bool {
655  const auto group = static_cast<InstrumentType>(static_cast<uint64_t>(type.first) & 0xFF00);
656  return (group == instrument.first) && (instrument.second == type.second);
657  })
658  != _instruments.end();
659  };
660 
661  for(const auto &instrument : _available_instruments)
662  {
663  if(all_instruments || is_selected(instrument.first))
664  {
665  profiler.add(instrument.second());
666  }
667  }
668 
669  return profiler;
670 }
671 
673 {
674  _printers.push_back(printer);
675 }
676 
677 std::vector<TestInfo> Framework::test_infos() const
678 {
679  std::vector<TestInfo> ids;
680 
681  int id = 0;
682 
683  for(const auto &factory : _test_factories)
684  {
685  const TestInfo test_info{ id, factory->name(), factory->mode(), factory->status() };
686 
687  if(_test_filter->is_selected(test_info))
688  {
689  ids.emplace_back(std::move(test_info));
690  }
691 
692  ++id;
693  }
694 
695  return ids;
696 }
697 
699 {
700  return _log_level;
701 }
702 
704 {
705  ARM_COMPUTE_ERROR_ON(instruments_info == nullptr);
706  *instruments_info = instr_info;
707 }
708 
710 {
711  return _configure_only;
712 }
713 
715 {
716  return _new_fixture_call;
717 }
718 
720 {
721  _new_fixture_call = val;
722 }
723 } // namespace framework
724 } // namespace test
725 } // namespace arm_compute
void pop_suite()
Remove innermost test suite.
Definition: Framework.cpp:151
virtual std::unique_ptr< TestCase > make() const =0
Factory function to create the test case.
Framework configuration structure.
Definition: Framework.h:57
std::string join(T first, T last, const std::string &separator)
Helper function to concatenate multiple strings.
Definition: Utils.h:93
virtual void print_error(const std::exception &error, bool expected)=0
Print test error.
virtual void print_run_footer()=0
Print footer after running all tests.
bool new_fixture_call() const
Return whether the new fixture has been called.
Definition: Framework.cpp:714
virtual void print_errors_header()=0
Print header before errors.
void add(std::unique_ptr< Instrument > instrument)
Add instrument to the performance monitor.
Definition: Profiler.cpp:35
int num_iterations
Number of iterations per test.
Definition: Framework.h:63
std::pair< InstrumentType, ScaleFactor > InstrumentsDescription
Definition: Instruments.h:69
static CLScheduler & get()
Access the scheduler singleton.
Profiler class to collect benchmark numbers.
Definition: Profiler.h:45
void set_context(cl::Context context)
Accessor to set the CL context to be used by the scheduler.
virtual void print_info(const std::string &info)=0
Print test log info.
void print_test_info(std::ostream &os) const
Print test info.
Definition: Framework.cpp:171
void set_throw_errors(bool throw_errors)
Set whether errors are caught or thrown by the framework.
Definition: Framework.cpp:265
bool stop_on_error() const
Indicates if test execution is stopped after the first failed test.
Definition: Framework.cpp:280
void log_failed_expectation(const TestError &error)
Tell the framework that the currently running test case failed a non-fatal expectation.
Definition: Framework.cpp:226
DatasetMode mode() const
Get the mode for which test case will be enabled.
Abstract printer class used by the Framework to present output.
Definition: Printer.h:43
void set_instruments_info(InstrumentsInfo instr_info)
Sets instruments info.
Definition: Framework.cpp:703
std::unique_ptr< InstrumentsInfo > instruments_info
Definition: Framework.cpp:50
void log_info(const std::string &info)
Print the debug information that has already been logged.
Definition: Framework.cpp:244
#define ARM_COMPUTE_ERROR_ON(cond)
If the condition is true then an error message is printed and an exception thrown.
Definition: Error.h:466
void test_stop()
Call test_stop() on all the added instruments.
Definition: Profiler.cpp:71
std::string header_data
Test header data.
Definition: TestResult.h:76
static CLKernelLibrary & get()
Access the KernelLibrary singleton.
bool throw_errors() const
Should errors be caught or thrown by the framework.
Definition: Framework.cpp:270
void set_error_on_missing_assets(bool error_on_missing_assets)
Set whether a test should be considered as failed if its assets cannot be found.
Definition: Framework.cpp:285
LogLevel
Severity of the information.
Definition: Exceptions.h:50
void set_test_result(TestInfo info, TestResult result)
Set the result for an executed test case.
Definition: Framework.cpp:623
decltype(strategy::transforms) typedef type
bool configure_only() const
Get the configure only flag.
Definition: Framework.cpp:709
Copyright (c) 2017-2021 Arm Limited.
float cooldown_sec
Delay between tests in seconds.
Definition: Framework.h:64
static Framework & get()
Access to the singleton.
Definition: Framework.cpp:124
Interface to enqueue OpenCL kernels and get/set the OpenCL CommandQueue and ICLTuner.
Abstract factory class to create test cases.
std::string name
Test name.
Definition: Framework.h:81
std::string name() const
Name of the test case.
LogLevel log_level
Verbosity of the output.
Definition: Framework.h:65
bool run()
Run all enabled test cases.
Definition: Framework.cpp:543
void clear_programs_cache()
Clear the library&#39;s cache of binary programs.
std::vector< framework::InstrumentsDescription > instruments
Instrument types that will be used for benchmarking.
Definition: Framework.h:59
LogLevel level() const
Severity of the error.
Definition: Exceptions.cpp:121
cl::Context & context()
Accessor for the associated CL context.
Definition: CLScheduler.cpp:32
std::vector< TestInfo > test_infos() const
List of TestInfo&#39;s.
Definition: Framework.cpp:677
std::string name_filter
Regular expression to filter tests by name.
Definition: Framework.h:60
std::unique_ptr< ParametersLibrary > parameters
Definition: Framework.cpp:46
const std::string & header() const
Return JSON formatted header data.
Definition: Profiler.cpp:94
const BatchNormalizationSelectorPtr is_selected
Status status() const
Get the status of the test case.
Error class for when some external assets are missing.
Definition: Exceptions.h:67
std::set< InstrumentsDescription > available_instruments() const
Supported instrument types for benchmarking.
Definition: Framework.cpp:100
Information about a test case.
Definition: Framework.h:78
void end(TokenStream &in, bool &valid)
Definition: MLGOParser.cpp:290
LogLevel log_level() const
Get the current logging level.
Definition: Framework.cpp:698
Profiler get_profiler() const
Factory method to obtain a configured profiler.
Definition: Framework.cpp:643
const char * name
cl::CommandQueue & queue()
Accessor for the associated CL command queue.
Definition: CLScheduler.cpp:39
void log_test_start(const TestInfo &info)
Tell the framework that execution of a test starts.
Definition: Framework.cpp:190
const MeasurementsMap & measurements() const
Return measurements for all instruments.
Definition: Profiler.cpp:89
void set_queue(cl::CommandQueue queue)
Accessor to set the CL command queue to be used by the scheduler.
Definition: CLScheduler.cpp:55
void for_each(F &&)
Base case of for_each.
Definition: Utility.h:110
Profiler::MeasurementsMap measurements
Profiling information.
Definition: TestResult.h:75
virtual void print_run_header()=0
Print header before running all tests.
ScaleKernelInfo info(interpolation_policy, default_border_mode, PixelValue(), sampling_policy, false)
void log_test_end(const TestInfo &info)
Tell the framework that a test case finished.
Definition: Framework.cpp:206
void add_test_info(std::string info)
Add info string for the next expectation/assertion.
Definition: Framework.cpp:156
bool error_on_missing_assets() const
Indicates if a test should be marked as failed when its assets are missing.
Definition: Framework.cpp:290
virtual void print_test_footer()=0
Print footer after a test.
std::string id_filter
String to match selected test ids.
Definition: Framework.h:61
bool has_test_info() const
Check if any info has been registered.
Definition: Framework.cpp:166
void set_num_iterations(int num_iterations)
Set number of iterations per test case.
Definition: Framework.cpp:260
void push_suite(std::string name)
Add a new test suite.
Definition: Framework.cpp:146
void start()
Call start() on all the added instruments.
Definition: Profiler.cpp:48
virtual void print_profiler_header(const std::string &header_data)=0
Print header data.
Class to store results of a test.
Definition: TestResult.h:39
Status status
Execution status.
Definition: TestResult.h:74
virtual void print_errors_footer()=0
Print footer after errors.
void sleep_in_seconds(float seconds)
Makes the calling thread to sleep for a specified number of seconds.
Definition: Utils.cpp:38
bool configure_only
Only configure kernels.
Definition: Framework.h:66
virtual void print_test_header(const TestInfo &info)=0
Print header before a test.
void add_printer(Printer *printer)
Set the printer used for the output of test results.
Definition: Framework.cpp:672
void stop()
Call stop() on all the added instruments.
Definition: Profiler.cpp:56
TestCaseFactory::Status status
Test status.
Definition: Framework.h:83
void test_start()
Call test_start() on all the added instruments.
Definition: Profiler.cpp:40
void log_test_skipped(const TestInfo &info)
Tell the framework that a test case is skipped.
Definition: Framework.cpp:201
void set_new_fixture_call(bool val)
Set the new fixture call flag.
Definition: Framework.cpp:719
Error class for failures during test execution.
Definition: Exceptions.h:78
void clear_test_info()
Clear the collected test info.
Definition: Framework.cpp:161
void set_stop_on_error(bool stop_on_error)
Set whether to abort execution after the first failed test.
Definition: Framework.cpp:275
int num_iterations() const
Number of iterations per test case.
Definition: Framework.cpp:255
void init(const FrameworkConfig &config)
Init the framework.
Definition: Framework.cpp:130
virtual void print_measurements(const Profiler::MeasurementsMap &measurements)=0
Print measurements for a test.
void print_test_results(Printer &printer) const
Use the specified printer to output test results from the last run.
Definition: Framework.cpp:628
bool opencl_is_available()
Check if OpenCL is available.
Definition: OpenCL.cpp:154