40 #include <type_traits> 52 Framework::Framework()
53 : _test_filter(nullptr)
57 Instrument::make_instrument<WallClockTimestamps, ScaleFactor::TIME_MS>);
59 Instrument::make_instrument<WallClockTimestamps, ScaleFactor::TIME_S>);
65 Instrument::make_instrument<SchedulerTimestamps, ScaleFactor::TIME_MS>);
67 Instrument::make_instrument<SchedulerTimestamps, ScaleFactor::TIME_S>);
92 Instrument::make_instrument<OpenCLMemoryUsage, ScaleFactor::SCALE_1K>);
94 Instrument::make_instrument<OpenCLMemoryUsage, ScaleFactor::SCALE_1M>);
100 std::set<InstrumentsDescription> Framework::available_instruments()
const 102 std::set<InstrumentsDescription> types;
104 for(
const auto &instrument : _available_instruments)
106 types.emplace(instrument.first);
112 std::map<TestResult::Status, int> Framework::count_test_results()
const 114 std::map<TestResult::Status, int> counts;
116 for(
const auto &test : _test_results)
118 ++counts[test.second.status];
141 std::string Framework::current_suite_name()
const 143 return join(_test_suite_name.cbegin(), _test_suite_name.cend(),
"/");
146 void Framework::push_suite(std::string
name)
148 _test_suite_name.emplace_back(std::move(
name));
151 void Framework::pop_suite()
153 _test_suite_name.pop_back();
156 void Framework::add_test_info(std::string
info)
158 _test_info.emplace_back(std::move(
info));
161 void Framework::clear_test_info()
166 bool Framework::has_test_info()
const 168 return !_test_info.empty();
171 void Framework::print_test_info(std::ostream &os)
const 173 if(!_test_info.empty())
177 for(
const auto &
str : _test_info)
179 os <<
" " <<
str <<
"\n";
184 template <
typename F>
185 void Framework::func_on_all_printers(F &&
func)
192 if(_log_level >= LogLevel::TESTS)
194 func_on_all_printers([&](
Printer * p)
203 static_cast<void>(
info);
208 if(_log_level >= LogLevel::MEASUREMENTS)
210 func_on_all_printers([&](
Printer * p)
216 if(_log_level >= LogLevel::TESTS)
218 func_on_all_printers([](
Printer * p)
225 void Framework::log_failed_expectation(
const TestError &error)
230 const bool is_expected_failure = _current_test_info->status == TestCaseFactory::Status::EXPECTED_FAILURE;
232 if(_log_level >= error.
level())
234 func_on_all_printers([&](
Printer * p)
240 _current_test_result->status = TestResult::Status::FAILED;
243 void Framework::log_info(
const std::string &
info)
245 if(_log_level >= LogLevel::DEBUG)
247 func_on_all_printers([&](
Printer * p)
254 int Framework::num_iterations()
const 256 return _num_iterations;
259 void Framework::set_num_iterations(
int num_iterations)
261 _num_iterations = num_iterations;
264 void Framework::set_throw_errors(
bool throw_errors)
266 _throw_errors = throw_errors;
269 bool Framework::throw_errors()
const 271 return _throw_errors;
274 void Framework::set_stop_on_error(
bool stop_on_error)
276 _stop_on_error = stop_on_error;
279 bool Framework::stop_on_error()
const 281 return _stop_on_error;
284 void Framework::set_error_on_missing_assets(
bool error_on_missing_assets)
286 _error_on_missing_assets = error_on_missing_assets;
289 bool Framework::error_on_missing_assets()
const 291 return _error_on_missing_assets;
296 if(test_factory.
status() == TestCaseFactory::Status::DISABLED)
298 log_test_skipped(
info);
303 log_test_start(
info);
305 Profiler profiler = get_profiler();
306 TestResult result(TestResult::Status::NOT_RUN);
308 _current_test_info = &
info;
309 _current_test_result = &result;
311 if(_log_level >= LogLevel::ERRORS)
313 func_on_all_printers([](Printer * p)
315 p->print_errors_header();
319 const bool is_expected_failure =
info.status == TestCaseFactory::Status::EXPECTED_FAILURE;
323 std::unique_ptr<TestCase> test_case = test_factory.
make();
327 profiler.test_start();
329 test_case->do_setup();
331 for(
int i = 0; i < _num_iterations; ++i)
339 if(_num_iterations == 1 || i != 0)
344 test_case->do_sync();
345 if(_num_iterations == 1 || i != 0)
351 test_case->do_teardown();
353 profiler.test_stop();
356 if(result.status == TestResult::Status::NOT_RUN)
358 result.status = TestResult::Status::SUCCESS;
361 catch(
const FileNotFound &error)
363 profiler.test_stop();
364 if(_error_on_missing_assets)
366 if(_log_level >= LogLevel::ERRORS)
368 TestError test_error(error.what(), LogLevel::ERRORS);
369 func_on_all_printers([&](Printer * p)
371 p->print_error(test_error, is_expected_failure);
375 result.status = TestResult::Status::FAILED;
384 if(_log_level >= LogLevel::DEBUG)
386 func_on_all_printers([&](Printer * p)
388 p->print_info(error.what());
392 result.status = TestResult::Status::NOT_RUN;
395 catch(
const TestError &error)
397 profiler.test_stop();
398 if(_log_level >= error.level())
400 func_on_all_printers([&](Printer * p)
402 p->print_error(error, is_expected_failure);
406 result.status = TestResult::Status::FAILED;
413 #ifdef ARM_COMPUTE_CL 414 catch(const ::cl::Error &error)
416 profiler.test_stop();
417 if(_log_level >= LogLevel::ERRORS)
419 std::stringstream stream;
420 stream <<
"Error code: " << error.err();
421 TestError test_error(error.what(), LogLevel::ERRORS, stream.str());
422 func_on_all_printers([&](Printer * p)
424 p->print_error(test_error, is_expected_failure);
428 result.status = TestResult::Status::FAILED;
436 catch(
const std::exception &error)
438 profiler.test_stop();
439 if(_log_level >= LogLevel::ERRORS)
441 func_on_all_printers([&](Printer * p)
443 p->print_error(error, is_expected_failure);
447 result.status = TestResult::Status::CRASHED;
456 profiler.test_stop();
457 if(_log_level >= LogLevel::ERRORS)
459 func_on_all_printers([&](Printer * p)
461 p->print_error(TestError(
"Received unknown exception"), is_expected_failure);
465 result.status = TestResult::Status::CRASHED;
473 catch(
const std::exception &error)
475 if(_log_level >= LogLevel::ERRORS)
477 func_on_all_printers([&](Printer * p)
479 p->print_error(error, is_expected_failure);
483 result.status = TestResult::Status::CRASHED;
492 if(_log_level >= LogLevel::ERRORS)
494 func_on_all_printers([&](Printer * p)
496 p->print_error(TestError(
"Received unknown exception"), is_expected_failure);
500 result.status = TestResult::Status::CRASHED;
508 if(_log_level >= LogLevel::ERRORS)
510 func_on_all_printers([](Printer * p)
512 p->print_errors_footer();
516 _current_test_info =
nullptr;
517 _current_test_result =
nullptr;
519 if(result.status == TestResult::Status::FAILED)
521 if(
info.status == TestCaseFactory::Status::EXPECTED_FAILURE)
523 result.status = TestResult::Status::EXPECTED_FAILURE;
527 if(result.status == TestResult::Status::FAILED || result.status == TestResult::Status::CRASHED)
531 throw std::runtime_error(
"Abort on first error.");
535 result.measurements = profiler.measurements();
537 set_test_result(
info, result);
544 _test_results.clear();
546 if(_log_level >= LogLevel::TESTS)
548 func_on_all_printers([](
Printer * p)
554 const std::chrono::time_point<std::chrono::high_resolution_clock> start = std::chrono::high_resolution_clock::now();
559 for(
auto &test_factory : _test_factories)
561 const std::string test_case_name = test_factory->
name();
562 const TestInfo test_info{ id, test_case_name, test_factory->
mode(), test_factory->
status() };
564 if(_test_filter->is_selected(test_info))
566 #ifdef ARM_COMPUTE_CL 573 cl::Context new_ctx = cl::Context(CL_DEVICE_TYPE_DEFAULT, ctx_properties.data());
574 cl::CommandQueue new_queue = cl::CommandQueue(new_ctx,
CLKernelLibrary::get().get_device(), queue_properties);
580 #endif // ARM_COMPUTE_CL 582 run_test(test_info, *test_factory);
593 const std::chrono::time_point<std::chrono::high_resolution_clock>
end = std::chrono::high_resolution_clock::now();
595 if(_log_level >= LogLevel::TESTS)
597 func_on_all_printers([](
Printer * p)
603 auto runtime = std::chrono::duration_cast<std::chrono::seconds>(
end - start);
604 std::map<TestResult::Status, int> results = count_test_results();
608 std::cout <<
"Executed " << _test_results.size() <<
" test(s) (" 609 << results[TestResult::Status::SUCCESS] <<
" passed, " 610 << results[TestResult::Status::EXPECTED_FAILURE] <<
" expected failures, " 611 << results[TestResult::Status::FAILED] <<
" failed, " 612 << results[TestResult::Status::CRASHED] <<
" crashed, " 613 << results[TestResult::Status::DISABLED] <<
" disabled) in " << runtime.count() <<
" second(s)\n";
616 int num_successful_tests = results[TestResult::Status::SUCCESS] + results[TestResult::Status::EXPECTED_FAILURE] + results[TestResult::Status::DISABLED];
618 return (static_cast<unsigned int>(num_successful_tests) == _test_results.size());
623 _test_results.emplace(std::move(
info), std::move(result));
626 void Framework::print_test_results(
Printer &printer)
const 630 for(
const auto &test : _test_results)
644 const bool all_instruments = std::any_of(
645 _instruments.begin(),
652 const auto group = static_cast<InstrumentType>(static_cast<uint64_t>(
type.first) & 0xFF00);
653 return (group == instrument.first) && (instrument.second ==
type.second);
655 != _instruments.end();
658 for(
const auto &instrument : _available_instruments)
660 if(all_instruments ||
is_selected(instrument.first))
662 profiler.
add(instrument.second());
671 _printers.push_back(printer);
674 std::vector<TestInfo> Framework::test_infos()
const 676 std::vector<TestInfo> ids;
680 for(
const auto &factory : _test_factories)
682 TestInfo test_info{ id, factory->
name(), factory->mode(), factory->status() };
684 if(_test_filter->is_selected(test_info))
686 ids.emplace_back(std::move(test_info));
706 bool Framework::configure_only()
const 708 return _configure_only;
711 bool Framework::new_fixture_call()
const 713 return _new_fixture_call;
716 void Framework::set_new_fixture_call(
bool val)
718 _new_fixture_call = val;
virtual std::unique_ptr< TestCase > make() const =0
Factory function to create the test case.
Framework configuration structure.
std::string join(T first, T last, const std::string &separator)
Helper function to concatenate multiple strings.
virtual void print_error(const std::exception &error, bool expected)=0
Print test error.
virtual void print_run_footer()=0
Print footer after running all tests.
void add(std::unique_ptr< Instrument > instrument)
Add instrument to the performance monitor.
int num_iterations
Number of iterations per test.
std::pair< InstrumentType, ScaleFactor > InstrumentsDescription
static CLScheduler & get()
Access the scheduler singleton.
Profiler class to collect benchmark numbers.
void set_context(cl::Context context)
Accessor to set the CL context to be used by the scheduler.
virtual void print_info(const std::string &info)=0
Print test log info.
const ActivationSelectorPtr is_selected
void sleep_in_seconds(float seconds)
Makes the calling thread to sleep for a specified number of seconds.
DatasetMode mode() const
Get the mode for which test case will be enabled.
Abstract printer class used by the Framework to present output.
std::unique_ptr< InstrumentsInfo > instruments_info
#define ARM_COMPUTE_ERROR_ON(cond)
If the condition is true then an error message is printed and an exception thrown.
static CLKernelLibrary & get()
Access the KernelLibrary singleton.
LogLevel
Severity of the information.
decltype(strategy::transforms) typedef type
Copyright (c) 2017-2021 Arm Limited.
float cooldown_sec
Delay between tests in seconds.
Interface to enqueue OpenCL kernels and get/set the OpenCL CommandQueue and ICLTuner.
Abstract factory class to create test cases.
std::string name
Test name.
std::string name() const
Name of the test case.
LogLevel log_level
Verbosity of the output.
void clear_programs_cache()
Clear the library's cache of binary programs.
std::vector< framework::InstrumentsDescription > instruments
Instrument types that will be used for benchmarking.
LogLevel level() const
Severity of the error.
cl::Context & context()
Accessor for the associated CL context.
std::string name_filter
Regular expression to filter tests by name.
std::unique_ptr< ParametersLibrary > parameters
Status status() const
Get the status of the test case.
DatasetMode mode
Dataset mode.
Information about a test case.
void end(TokenStream &in, bool &valid)
cl::CommandQueue & queue()
Accessor for the associated CL command queue.
void set_queue(cl::CommandQueue queue)
Accessor to set the CL command queue to be used by the scheduler.
void for_each(F &&)
Base case of for_each.
virtual void print_run_header()=0
Print header before running all tests.
ScaleKernelInfo info(interpolation_policy, default_border_mode, PixelValue(), sampling_policy, false)
virtual void print_test_footer()=0
Print footer after a test.
std::string id_filter
String to match selected test ids.
Class to store results of a test.
bool configure_only
Only configure kernels.
virtual void print_test_header(const TestInfo &info)=0
Print header before a test.
Error class for failures during test execution.
virtual void print_measurements(const Profiler::MeasurementsMap &measurements)=0
Print measurements for a test.
bool opencl_is_available()
Check if OpenCL is available.