40 #include <type_traits>
52 Framework::Framework()
53 : _test_filter(nullptr)
57 Instrument::make_instrument<WallClockTimestamps, ScaleFactor::TIME_MS>);
59 Instrument::make_instrument<WallClockTimestamps, ScaleFactor::TIME_S>);
65 Instrument::make_instrument<SchedulerTimestamps, ScaleFactor::TIME_MS>);
67 Instrument::make_instrument<SchedulerTimestamps, ScaleFactor::TIME_S>);
92 Instrument::make_instrument<OpenCLMemoryUsage, ScaleFactor::SCALE_1K>);
94 Instrument::make_instrument<OpenCLMemoryUsage, ScaleFactor::SCALE_1M>);
100 std::set<InstrumentsDescription> Framework::available_instruments()
const
102 std::set<InstrumentsDescription> types;
104 for(
const auto &instrument : _available_instruments)
106 types.emplace(instrument.first);
112 std::map<TestResult::Status, int> Framework::count_test_results()
const
114 std::map<TestResult::Status, int> counts;
116 for(
const auto &test : _test_results)
118 ++counts[test.second.status];
143 std::string Framework::current_suite_name()
const
145 return join(_test_suite_name.cbegin(), _test_suite_name.cend(),
"/");
148 void Framework::push_suite(std::string
name)
150 _test_suite_name.emplace_back(std::move(
name));
153 void Framework::pop_suite()
155 _test_suite_name.pop_back();
158 void Framework::add_test_info(std::string
info)
160 _test_info.emplace_back(std::move(
info));
163 void Framework::clear_test_info()
168 bool Framework::has_test_info()
const
170 return !_test_info.empty();
173 void Framework::print_test_info(std::ostream &os)
const
175 if(!_test_info.empty())
179 for(
const auto &
str : _test_info)
181 os <<
" " <<
str <<
"\n";
186 template <
typename F>
187 void Framework::func_on_all_printers(F &&func)
194 if(_log_level >= LogLevel::TESTS)
196 func_on_all_printers([&](
Printer * p)
205 static_cast<void>(
info);
210 if(_log_level >= LogLevel::MEASUREMENTS)
212 func_on_all_printers([&](
Printer * p)
219 if(_log_level >= LogLevel::TESTS)
221 func_on_all_printers([](
Printer * p)
228 void Framework::log_failed_expectation(
const TestError &error)
233 const bool is_expected_failure = _current_test_info->status == TestCaseFactory::Status::EXPECTED_FAILURE;
235 if(_log_level >= error.
level())
237 func_on_all_printers([&](
Printer * p)
243 _current_test_result->status = TestResult::Status::FAILED;
246 void Framework::log_info(
const std::string &
info)
248 if(_log_level >= LogLevel::DEBUG)
250 func_on_all_printers([&](
Printer * p)
257 int Framework::num_iterations()
const
259 return _num_iterations;
262 void Framework::set_num_iterations(
int num_iterations)
264 _num_iterations = num_iterations;
267 void Framework::set_throw_errors(
bool throw_errors)
269 _throw_errors = throw_errors;
272 bool Framework::throw_errors()
const
274 return _throw_errors;
277 void Framework::set_stop_on_error(
bool stop_on_error)
279 _stop_on_error = stop_on_error;
282 bool Framework::stop_on_error()
const
284 return _stop_on_error;
287 void Framework::set_error_on_missing_assets(
bool error_on_missing_assets)
289 _error_on_missing_assets = error_on_missing_assets;
292 bool Framework::error_on_missing_assets()
const
294 return _error_on_missing_assets;
299 if(test_factory.
status() == TestCaseFactory::Status::DISABLED)
301 log_test_skipped(
info);
303 return TestResult::Status::DISABLED;
306 log_test_start(
info);
308 Profiler profiler = get_profiler();
309 TestResult result(TestResult::Status::NOT_RUN);
311 _current_test_info = &
info;
312 _current_test_result = &result;
314 if(_log_level >= LogLevel::ERRORS)
316 func_on_all_printers([](Printer * p)
318 p->print_errors_header();
322 const bool is_expected_failure =
info.status == TestCaseFactory::Status::EXPECTED_FAILURE;
326 std::unique_ptr<TestCase> test_case = test_factory.
make();
330 profiler.test_start();
332 test_case->do_setup();
334 for(
int i = 0; i < _num_iterations; ++i)
342 if(_num_iterations == 1 || i != 0)
347 test_case->do_sync();
348 if(_num_iterations == 1 || i != 0)
354 test_case->do_teardown();
356 profiler.test_stop();
359 if(result.status == TestResult::Status::NOT_RUN)
361 result.status = TestResult::Status::SUCCESS;
364 catch(
const FileNotFound &error)
366 profiler.test_stop();
367 if(_error_on_missing_assets)
369 if(_log_level >= LogLevel::ERRORS)
371 TestError test_error(error.what(), LogLevel::ERRORS);
372 func_on_all_printers([&](Printer * p)
374 p->print_error(test_error, is_expected_failure);
378 result.status = TestResult::Status::FAILED;
387 if(_log_level >= LogLevel::DEBUG)
389 func_on_all_printers([&](Printer * p)
391 p->print_info(error.what());
395 result.status = TestResult::Status::NOT_RUN;
398 catch(
const TestError &error)
400 profiler.test_stop();
401 if(_log_level >= error.level())
403 func_on_all_printers([&](Printer * p)
405 p->print_error(error, is_expected_failure);
409 result.status = TestResult::Status::FAILED;
416 #ifdef ARM_COMPUTE_CL
417 catch(const ::cl::Error &error)
419 profiler.test_stop();
420 if(_log_level >= LogLevel::ERRORS)
422 std::stringstream stream;
423 stream <<
"Error code: " << error.err();
424 TestError test_error(error.what(), LogLevel::ERRORS, stream.str());
425 func_on_all_printers([&](Printer * p)
427 p->print_error(test_error, is_expected_failure);
431 result.status = TestResult::Status::FAILED;
439 catch(
const std::exception &error)
441 profiler.test_stop();
442 if(_log_level >= LogLevel::ERRORS)
444 func_on_all_printers([&](Printer * p)
446 p->print_error(error, is_expected_failure);
450 result.status = TestResult::Status::CRASHED;
459 profiler.test_stop();
460 if(_log_level >= LogLevel::ERRORS)
462 func_on_all_printers([&](Printer * p)
464 p->print_error(TestError(
"Received unknown exception"), is_expected_failure);
468 result.status = TestResult::Status::CRASHED;
476 catch(
const std::exception &error)
478 if(_log_level >= LogLevel::ERRORS)
480 func_on_all_printers([&](Printer * p)
482 p->print_error(error, is_expected_failure);
486 result.status = TestResult::Status::CRASHED;
495 if(_log_level >= LogLevel::ERRORS)
497 func_on_all_printers([&](Printer * p)
499 p->print_error(TestError(
"Received unknown exception"), is_expected_failure);
503 result.status = TestResult::Status::CRASHED;
511 if(_log_level >= LogLevel::ERRORS)
513 func_on_all_printers([](Printer * p)
515 p->print_errors_footer();
519 _current_test_info =
nullptr;
520 _current_test_result =
nullptr;
522 if(result.status == TestResult::Status::FAILED)
524 if(
info.status == TestCaseFactory::Status::EXPECTED_FAILURE)
526 result.status = TestResult::Status::EXPECTED_FAILURE;
530 if(result.status == TestResult::Status::FAILED || result.status == TestResult::Status::CRASHED)
534 throw std::runtime_error(
"Abandon on first error.");
538 result.header_data = profiler.header();
539 result.measurements = profiler.measurements();
541 set_test_result(
info, result);
543 return result.status;
549 _test_results.clear();
551 if(_log_level >= LogLevel::TESTS)
553 func_on_all_printers([](
Printer * p)
559 const std::chrono::time_point<std::chrono::high_resolution_clock> start = std::chrono::high_resolution_clock::now();
565 for(
auto &test_factory : _test_factories)
567 const std::string test_case_name = test_factory->
name();
568 const TestInfo test_info{ id, test_case_name, test_factory->
mode(), test_factory->
status() };
570 if(_test_filter->is_selected(test_info))
572 #ifdef ARM_COMPUTE_CL
579 cl::Context new_ctx = cl::Context(CL_DEVICE_TYPE_DEFAULT, ctx_properties.data());
580 cl::CommandQueue new_queue = cl::CommandQueue(new_ctx,
CLKernelLibrary::get().get_device(), queue_properties);
586 #endif // ARM_COMPUTE_CL
588 if((_print_rerun_cmd) && (result == TestResult::Status::CRASHED || result == TestResult::Status::FAILED))
590 std::cout <<
"Rerun command: ./arm_compute_validation --filter='^" << test_info.name <<
"$' --seed=" << _seed << std::endl;
601 const std::chrono::time_point<std::chrono::high_resolution_clock>
end = std::chrono::high_resolution_clock::now();
603 if(_log_level >= LogLevel::TESTS)
605 func_on_all_printers([](
Printer * p)
611 auto runtime = std::chrono::duration_cast<std::chrono::seconds>(
end - start);
612 std::map<TestResult::Status, int> results = count_test_results();
614 if(_log_level > LogLevel::NONE)
616 std::cout <<
"Executed " << _test_results.size() <<
" test(s) ("
617 << results[TestResult::Status::SUCCESS] <<
" passed, "
618 << results[TestResult::Status::EXPECTED_FAILURE] <<
" expected failures, "
619 << results[TestResult::Status::FAILED] <<
" failed, "
620 << results[TestResult::Status::CRASHED] <<
" crashed, "
621 << results[TestResult::Status::DISABLED] <<
" disabled) in " << runtime.count() <<
" second(s)\n";
624 int num_successful_tests = results[TestResult::Status::SUCCESS] + results[TestResult::Status::EXPECTED_FAILURE] + results[TestResult::Status::DISABLED];
626 return (
static_cast<unsigned int>(num_successful_tests) == _test_results.size());
631 _test_results.emplace(std::move(
info), std::move(result));
634 void Framework::print_test_results(
Printer &printer)
const
638 for(
const auto &test : _test_results)
653 const bool all_instruments = std::any_of(
654 _instruments.begin(),
661 const auto group = static_cast<InstrumentType>(static_cast<uint64_t>(type.first) & 0xFF00);
662 return (group == instrument.first) && (instrument.second == type.second);
664 != _instruments.end();
667 for(
const auto &instrument : _available_instruments)
669 if(all_instruments ||
is_selected(instrument.first))
671 profiler.
add(instrument.second());
680 _printers.push_back(printer);
683 std::vector<TestInfo> Framework::test_infos()
const
685 std::vector<TestInfo> ids;
689 for(
const auto &factory : _test_factories)
691 const TestInfo test_info{ id, factory->
name(), factory->mode(), factory->status() };
693 if(_test_filter->is_selected(test_info))
695 ids.emplace_back(std::move(test_info));
715 bool Framework::configure_only()
const
717 return _configure_only;
720 bool Framework::new_fixture_call()
const
722 return _new_fixture_call;
725 void Framework::set_new_fixture_call(
bool val)
727 _new_fixture_call = val;