Main framework class. More...

#include <Framework.h>

Public Member Functions
std::set< InstrumentsDescription >	available_instruments () const
	Supported instrument types for benchmarking. More...

void	init (const FrameworkConfig &config)
	Init the framework. More...

void	push_suite (std::string name)
	Add a new test suite. More...

void	pop_suite ()
	Remove innermost test suite. More...

template<typename T >
void	add_test_case (std::string test_name, DatasetMode mode, TestCaseFactory::Status status)
	Add a test case to the framework. More...

template<typename T , typename D >
void	add_data_test_case (std::string test_name, DatasetMode mode, TestCaseFactory::Status status, std::string description, D &&data)
	Add a data test case to the framework. More...

void	add_test_info (std::string info)
	Add info string for the next expectation/assertion. More...

void	clear_test_info ()
	Clear the collected test info. More...

bool	has_test_info () const
	Check if any info has been registered. More...

void	print_test_info (std::ostream &os) const
	Print test info. More...

void	log_test_start (const TestInfo &info)
	Tell the framework that execution of a test starts. More...

void	log_test_skipped (const TestInfo &info)
	Tell the framework that a test case is skipped. More...

void	log_test_end (const TestInfo &info)
	Tell the framework that a test case finished. More...

void	log_failed_expectation (const TestError &error)
	Tell the framework that the currently running test case failed a non-fatal expectation. More...

void	log_info (const std::string &info)
	Print the debug information that has already been logged. More...

int	num_iterations () const
	Number of iterations per test case. More...

void	set_num_iterations (int num_iterations)
	Set number of iterations per test case. More...

bool	throw_errors () const
	Should errors be caught or thrown by the framework. More...

void	set_throw_errors (bool throw_errors)
	Set whether errors are caught or thrown by the framework. More...

bool	stop_on_error () const
	Indicates if test execution is stopped after the first failed test. More...

void	set_stop_on_error (bool stop_on_error)
	Set whether to stop execution after the first failed test. More...

bool	error_on_missing_assets () const
	Indicates if a test should be marked as failed when its assets are missing. More...

void	set_error_on_missing_assets (bool error_on_missing_assets)
	Set whether a test should be considered as failed if its assets cannot be found. More...

bool	run ()
	Run all enabled test cases. More...

void	set_test_result (TestInfo info, TestResult result)
	Set the result for an executed test case. More...

void	print_test_results (Printer &printer) const
	Use the specified printer to output test results from the last run. More...

Profiler	get_profiler () const
	Factory method to obtain a configured profiler. More...

void	add_printer (Printer *printer)
	Set the printer used for the output of test results. More...

std::vector< TestInfo >	test_infos () const
	List of TestInfo's. More...

LogLevel	log_level () const
	Get the current logging level. More...

void	set_instruments_info (InstrumentsInfo instr_info)
	Sets instruments info. More...

bool	configure_only () const
	Get the configure only flag. More...

bool	new_fixture_call () const
	Return whether the new fixture has been called. More...

void	set_new_fixture_call (bool val)
	Set the new fixture call flag. More...

Static Public Member Functions
static Framework &	get ()
	Access to the singleton. More...

Detailed Description

Main framework class.

Keeps track of the global state, owns all test cases and collects results.

Definition at line 97 of file Framework.h.

Member Function Documentation

◆ add_data_test_case()

void add_data_test_case	(	std::string	test_name,
		DatasetMode	mode,
		TestCaseFactory::Status	status,
		std::string	description,
		D &&	data
	)

inline

Add a data test case to the framework.

Parameters

[in]	test_name	Name of the new test case.
[in]	mode	Mode in which to include the test.
[in]	status	Status of the test case.
[in]	description	Description of `data`.
[in]	data	Data that will be used as input to the test.

Definition at line 382 of file Framework.h.

 {
     // WORKAROUND for GCC 4.9
     // The function should get *it which is tuple but that seems to trigger a
     // bug in the compiler.
     auto tmp = std::unique_ptr<DataTestCaseFactory<T, decltype(*std::declval<D>())>>(new DataTestCaseFactory<T, decltype(*std::declval<D>())>(current_suite_name(), std::move(test_name), mode, status,
                                                                                      std::move(description), *data));
     _test_factories.emplace_back(std::move(tmp));
 }

References clang_tidy_rules::mode.

Referenced by TestCaseRegistrar< T >::TestCaseRegistrar().

◆ add_printer()

void add_printer ( Printer * printer )

Set the printer used for the output of test results.

Parameters

[in] printer Pointer to a printer.

Definition at line 678 of file Framework.cpp.

 {
     _printers.push_back(printer);
 }

Referenced by main().

◆ add_test_case()

void add_test_case	(	std::string	test_name,
		DatasetMode	mode,
		TestCaseFactory::Status	status
	)

inline

Add a test case to the framework.

Parameters

[in]	test_name	Name of the new test case.
[in]	mode	Mode in which to include the test.
[in]	status	Status of the test case.

Definition at line 376 of file Framework.h.

 {
     _test_factories.emplace_back(std::make_unique<SimpleTestCaseFactory<T>>(current_suite_name(), std::move(test_name), mode, status));
 }

References clang_tidy_rules::mode.

Referenced by TestCaseRegistrar< T >::TestCaseRegistrar().

◆ add_test_info()

void add_test_info ( std::string info )

Add info string for the next expectation/assertion.

Parameters

[in] info Info string.

Definition at line 158 of file Framework.cpp.

 {
     _test_info.emplace_back(std::move(info));
 }

References arm_compute::test::validation::info.

◆ available_instruments()

std::set< InstrumentsDescription > available_instruments ( ) const

Supported instrument types for benchmarking.

Returns: Set of all available instrument types.

Definition at line 100 of file Framework.cpp.

 {
     std::set<InstrumentsDescription> types;
  
     for(const auto &instrument : _available_instruments)
     {
         types.emplace(instrument.first);
     }
  
     return types;
 }

Referenced by CommonOptions::CommonOptions().

◆ clear_test_info()

void clear_test_info ( )

Clear the collected test info.

Definition at line 163 of file Framework.cpp.

 {
     _test_info.clear();
 }

Referenced by arm_compute::test::framework::ARM_COMPUTE_PRINT_INFO().

◆ configure_only()

bool configure_only ( ) const

Get the configure only flag.

Returns: The current configure only flag.

Definition at line 715 of file Framework.cpp.

 {
     return _configure_only;
 }

◆ error_on_missing_assets()

bool error_on_missing_assets ( ) const

Indicates if a test should be marked as failed when its assets are missing.

Returns: True if a test should be marked as failed when its assets are missing.

Definition at line 292 of file Framework.cpp.

 {
     return _error_on_missing_assets;
 }

◆ get()

Framework & get ( )

static

Access to the singleton.

Returns: Unique instance of the framework class.

Definition at line 124 of file Framework.cpp.

 {
     static Framework instance;
     return instance;
 }

Referenced by arm_compute::test::framework::ARM_COMPUTE_PRINT_INFO(), CommonOptions::CommonOptions(), main(), arm_compute::utils::run_example(), TestCaseRegistrar< T >::TestCaseRegistrar(), TestSuiteRegistrar::TestSuiteRegistrar(), arm_compute::test::validation::validate(), arm_compute::test::validation::validate_min_max_loc(), arm_compute::test::validation::validate_wrap(), and CLAccessor::~CLAccessor().

◆ get_profiler()

Profiler get_profiler ( ) const

Factory method to obtain a configured profiler.

The profiler enables all instruments that have been passed to the init method.

Returns: Configured profiler to collect benchmark results.

Definition at line 649 of file Framework.cpp.

 {
     Profiler profiler;
  
     const bool all_instruments = std::any_of(
                                      _instruments.begin(),
                                      _instruments.end(),
                                      [](InstrumentsDescription type) -> bool { return type.first == InstrumentType::ALL; });
  
     auto is_selected = [&](InstrumentsDescription instrument) -> bool
     {
         return std::find_if(_instruments.begin(), _instruments.end(), [&](InstrumentsDescription type) -> bool {
             const auto group = static_cast<InstrumentType>(static_cast<uint64_t>(type.first) & 0xFF00);
             return (group == instrument.first) && (instrument.second == type.second);
         })
         != _instruments.end();
     };
  
     for(const auto &instrument : _available_instruments)
     {
         if(all_instruments || is_selected(instrument.first))
         {
             profiler.add(instrument.second());
         }
     }
  
     return profiler;
 }

References Profiler::add(), is_selected, and type.

Referenced by main().

◆ has_test_info()

bool has_test_info ( ) const

Check if any info has been registered.

Returns: True if there is test info.

Definition at line 168 of file Framework.cpp.

 {
     return !_test_info.empty();
 }

◆ init()

void init ( const FrameworkConfig & config )

Init the framework.

See also: TestFilter::TestFilter for the format of the string to filter ids.

Parameters

[in] config Framework configuration meta-data.

Definition at line 130 of file Framework.cpp.

 {
     _test_filter.reset(new TestFilter(config.mode, config.name_filter, config.id_filter));
     _num_iterations  = config.num_iterations;
     _log_level       = config.log_level;
     _cooldown_sec    = config.cooldown_sec;
     _configure_only  = config.configure_only;
     _print_rerun_cmd = config.print_rerun_cmd;
     _seed            = config.seed;
  
     _instruments = std::set<framework::InstrumentsDescription>(std::begin(config.instruments), std::end(config.instruments));
 }

References FrameworkConfig::configure_only, FrameworkConfig::cooldown_sec, arm_compute::mlgo::parser::end(), FrameworkConfig::id_filter, FrameworkConfig::instruments, FrameworkConfig::log_level, FrameworkConfig::mode, FrameworkConfig::name_filter, FrameworkConfig::num_iterations, FrameworkConfig::print_rerun_cmd, and FrameworkConfig::seed.

Referenced by main().

◆ log_failed_expectation()

void log_failed_expectation ( const TestError & error )

Tell the framework that the currently running test case failed a non-fatal expectation.

Parameters

[in] error Description of the error.

Definition at line 228 of file Framework.cpp.

 {
     ARM_COMPUTE_ERROR_ON(_current_test_info == nullptr);
     ARM_COMPUTE_ERROR_ON(_current_test_result == nullptr);
  
     const bool is_expected_failure = _current_test_info->status == TestCaseFactory::Status::EXPECTED_FAILURE;
  
     if(_log_level >= error.level())
     {
         func_on_all_printers([&](Printer * p)
         {
             p->print_error(error, is_expected_failure);
         });
     }
  
     _current_test_result->status = TestResult::Status::FAILED;
 }

References ARM_COMPUTE_ERROR_ON, TestError::level(), and Printer::print_error().

◆ log_info()

void log_info ( const std::string & info )

Print the debug information that has already been logged.

Parameters

[in] info Description of the log info.

Definition at line 246 of file Framework.cpp.

 {
     if(_log_level >= LogLevel::DEBUG)
     {
         func_on_all_printers([&](Printer * p)
         {
             p->print_info(info);
         });
     }
 }

References arm_compute::test::validation::info, and Printer::print_info().

Referenced by arm_compute::test::framework::ARM_COMPUTE_PRINT_INFO().

◆ log_level()

LogLevel log_level ( ) const

Get the current logging level.

Returns: The current logging level.

Definition at line 704 of file Framework.cpp.

 {
     return _log_level;
 }

◆ log_test_end()

void log_test_end ( const TestInfo & info )

Tell the framework that a test case finished.

Parameters

[in] info Test info.

Definition at line 208 of file Framework.cpp.

 {
     if(_log_level >= LogLevel::MEASUREMENTS)
     {
         func_on_all_printers([&](Printer * p)
         {
             p->print_profiler_header(_test_results.at(info).header_data);
             p->print_measurements(_test_results.at(info).measurements);
         });
     }
  
     if(_log_level >= LogLevel::TESTS)
     {
         func_on_all_printers([](Printer * p)
         {
             p->print_test_footer();
         });
     }
 }

References arm_compute::test::validation::info, Printer::print_measurements(), Printer::print_profiler_header(), and Printer::print_test_footer().

◆ log_test_skipped()

void log_test_skipped ( const TestInfo & info )

Tell the framework that a test case is skipped.

Parameters

[in] info Test info.

Definition at line 203 of file Framework.cpp.

 {
     static_cast<void>(info);
 }

References arm_compute::test::validation::info.

◆ log_test_start()

void log_test_start ( const TestInfo & info )

Tell the framework that execution of a test starts.

Parameters

[in] info Test info.

Definition at line 192 of file Framework.cpp.

 {
     if(_log_level >= LogLevel::TESTS)
     {
         func_on_all_printers([&](Printer * p)
         {
             p->print_test_header(info);
         });
     }
 }

References arm_compute::test::validation::info, and Printer::print_test_header().

◆ new_fixture_call()

bool new_fixture_call ( ) const

Return whether the new fixture has been called.

Returns: The current new fixture call flag.

Definition at line 720 of file Framework.cpp.

 {
     return _new_fixture_call;
 }

◆ num_iterations()

int num_iterations ( ) const

Number of iterations per test case.

Returns: Number of iterations per test case.

Definition at line 257 of file Framework.cpp.

 {
     return _num_iterations;
 }

◆ pop_suite()

void pop_suite ( )

Remove innermost test suite.

Warning: Cannot be used at execution time. It can only be used for registering test cases.

Definition at line 153 of file Framework.cpp.

 {
     _test_suite_name.pop_back();
 }

Referenced by TestSuiteRegistrar::TestSuiteRegistrar().

◆ print_test_info()

void print_test_info ( std::ostream & os ) const

Print test info.

Parameters

[out] os Output stream.

Definition at line 173 of file Framework.cpp.

 {
     if(!_test_info.empty())
     {
         os << "CONTEXT:\n";
  
         for(const auto &str : _test_info)
         {
             os << "    " << str << "\n";
         }
     }
 }

References caffe_mnist_image_extractor::str.

Referenced by arm_compute::test::framework::ARM_COMPUTE_PRINT_INFO().

◆ print_test_results()

void print_test_results ( Printer & printer ) const

Use the specified printer to output test results from the last run.

This method can be used if the test results need to be obtained using a different printer than the one managed by the framework.

Parameters

[in] printer Printer used to output results.

Definition at line 634 of file Framework.cpp.

 {
     printer.print_run_header();
  
     for(const auto &test : _test_results)
     {
         printer.print_test_header(test.first);
         printer.print_profiler_header(test.second.header_data);
         printer.print_measurements(test.second.measurements);
         printer.print_test_footer();
     }
  
     printer.print_run_footer();
 }

References Printer::print_measurements(), Printer::print_profiler_header(), Printer::print_run_footer(), Printer::print_run_header(), Printer::print_test_footer(), and Printer::print_test_header().

◆ push_suite()

void push_suite ( std::string name )

Add a new test suite.

Warning: Cannot be used at execution time. It can only be used for registering test cases.

Parameters

[in] name Name of the added test suite.

Definition at line 148 of file Framework.cpp.

 {
     _test_suite_name.emplace_back(std::move(name));
 }

References name.

Referenced by TestSuiteRegistrar::TestSuiteRegistrar().

◆ run()

bool run ( )

Run all enabled test cases.

Returns: True if all test cases executed successful.

Definition at line 546 of file Framework.cpp.

 {
     // Clear old test results
     _test_results.clear();
  
     if(_log_level >= LogLevel::TESTS)
     {
         func_on_all_printers([](Printer * p)
         {
             p->print_run_header();
         });
     }
  
     const std::chrono::time_point<std::chrono::high_resolution_clock> start = std::chrono::high_resolution_clock::now();
  
     int id          = 0;
     int id_run_test = 0;
     ARM_COMPUTE_UNUSED(id_run_test); // Not used if ARM_COMPUTE_CL is not defined
  
     for(auto &test_factory : _test_factories)
     {
         const std::string test_case_name = test_factory->name();
         const TestInfo    test_info{ id, test_case_name, test_factory->mode(), test_factory->status() };
  
         if(_test_filter->is_selected(test_info))
         {
 #ifdef ARM_COMPUTE_CL
             // Every 100 tests, reset the OpenCL context to release the allocated memory
             if(opencl_is_available() && (id_run_test % 100) == 0)
             {
                 auto ctx_properties   = CLScheduler::get().context().getInfo<CL_CONTEXT_PROPERTIES>(nullptr);
                 auto queue_properties = CLScheduler::get().queue().getInfo<CL_QUEUE_PROPERTIES>(nullptr);
  
                 cl::Context      new_ctx   = cl::Context(CL_DEVICE_TYPE_DEFAULT, ctx_properties.data());
                 cl::CommandQueue new_queue = cl::CommandQueue(new_ctx, CLKernelLibrary::get().get_device(), queue_properties);
  
                 CLKernelLibrary::get().clear_programs_cache();
                 CLScheduler::get().set_context(new_ctx);
                 CLScheduler::get().set_queue(new_queue);
             }
 #endif // ARM_COMPUTE_CL
             TestResult::Status result = run_test(test_info, *test_factory);
             if((_print_rerun_cmd) && (result == TestResult::Status::CRASHED || result == TestResult::Status::FAILED))
             {
                 std::cout << "Rerun command: ./arm_compute_validation --filter='^" << test_info.name << "$' --seed=" << _seed << std::endl;
             }
             ++id_run_test;
  
             // Run test delay
             sleep_in_seconds(_cooldown_sec);
         }
  
         ++id;
     }
  
     const std::chrono::time_point<std::chrono::high_resolution_clock> end = std::chrono::high_resolution_clock::now();
  
     if(_log_level >= LogLevel::TESTS)
     {
         func_on_all_printers([](Printer * p)
         {
             p->print_run_footer();
         });
     }
  
     auto runtime = std::chrono::duration_cast<std::chrono::seconds>(end - start);
     std::map<TestResult::Status, int> results = count_test_results();
  
     if(_log_level > LogLevel::NONE)
     {
         std::cout << "Executed " << _test_results.size() << " test(s) ("
                   << results[TestResult::Status::SUCCESS] << " passed, "
                   << results[TestResult::Status::EXPECTED_FAILURE] << " expected failures, "
                   << results[TestResult::Status::FAILED] << " failed, "
                   << results[TestResult::Status::CRASHED] << " crashed, "
                   << results[TestResult::Status::DISABLED] << " disabled) in " << runtime.count() << " second(s)\n";
     }
  
     int num_successful_tests = results[TestResult::Status::SUCCESS] + results[TestResult::Status::EXPECTED_FAILURE] + results[TestResult::Status::DISABLED];
  
     return (static_cast<unsigned int>(num_successful_tests) == _test_results.size());
 }

References ARM_COMPUTE_UNUSED, CLKernelLibrary::clear_programs_cache(), CLScheduler::context(), arm_compute::mlgo::parser::end(), CLKernelLibrary::get(), CLScheduler::get(), TestCaseFactory::mode(), TestCaseFactory::name(), arm_compute::opencl_is_available(), Printer::print_run_footer(), Printer::print_run_header(), CLScheduler::queue(), CLScheduler::set_context(), CLScheduler::set_queue(), arm_compute::test::sleep_in_seconds(), and TestCaseFactory::status().

Referenced by main().

◆ set_error_on_missing_assets()

void set_error_on_missing_assets ( bool error_on_missing_assets )

Set whether a test should be considered as failed if its assets cannot be found.

Parameters

[in] error_on_missing_assets True if a test should be marked as failed when its assets are missing.

Definition at line 287 of file Framework.cpp.

 {
     _error_on_missing_assets = error_on_missing_assets;
 }

Referenced by main().

◆ set_instruments_info()

void set_instruments_info ( InstrumentsInfo instr_info )

Sets instruments info.

Note: TODO(COMPMID-2638) : Remove once instruments are transferred outside the framework.

Parameters

[in] instr_info Instruments info to set

Definition at line 709 of file Framework.cpp.

 {
     ARM_COMPUTE_ERROR_ON(instruments_info == nullptr);
     *instruments_info = instr_info;
 }

References ARM_COMPUTE_ERROR_ON, and arm_compute::test::framework::instruments_info.

Referenced by main().

◆ set_new_fixture_call()

void set_new_fixture_call ( bool val )

Set the new fixture call flag.

Parameters

[in] val Value to set for the flag

Definition at line 725 of file Framework.cpp.

 {
     _new_fixture_call = val;
 }

◆ set_num_iterations()

void set_num_iterations ( int num_iterations )

Set number of iterations per test case.

Parameters

[in] num_iterations Number of iterations per test case.

Definition at line 262 of file Framework.cpp.

 {
     _num_iterations = num_iterations;
 }

◆ set_stop_on_error()

void set_stop_on_error ( bool stop_on_error )

Set whether to stop execution after the first failed test.

Parameters

[in] stop_on_error True if execution is going to be stopped after first failed test.

Definition at line 277 of file Framework.cpp.

 {
     _stop_on_error = stop_on_error;
 }

Referenced by main().

◆ set_test_result()

void set_test_result	(	TestInfo	info,
		TestResult	result
	)

Set the result for an executed test case.

Parameters

[in]	info	Test info.
[in]	result	Execution result.

Definition at line 629 of file Framework.cpp.

 {
     _test_results.emplace(std::move(info), std::move(result));
 }

References arm_compute::test::validation::info.

◆ set_throw_errors()

void set_throw_errors ( bool throw_errors )

Set whether errors are caught or thrown by the framework.

Parameters

[in] throw_errors True if errors should be thrown.

Definition at line 267 of file Framework.cpp.

 {
     _throw_errors = throw_errors;
 }

Referenced by main().

◆ stop_on_error()

bool stop_on_error ( ) const

Indicates if test execution is stopped after the first failed test.

Returns: True if the execution is going to be stopped after the first failed test.

Definition at line 282 of file Framework.cpp.

 {
     return _stop_on_error;
 }

◆ test_infos()

std::vector< TestInfo > test_infos ( ) const

List of TestInfo's.

Returns: Vector with all test ids.

Definition at line 683 of file Framework.cpp.

 {
     std::vector<TestInfo> ids;
  
     int id = 0;
  
     for(const auto &factory : _test_factories)
     {
         const TestInfo test_info{ id, factory->name(), factory->mode(), factory->status() };
  
         if(_test_filter->is_selected(test_info))
         {
             ids.emplace_back(std::move(test_info));
         }
  
         ++id;
     }
  
     return ids;
 }

References TestInfo::name.

Referenced by main().

◆ throw_errors()

bool throw_errors ( ) const

Should errors be caught or thrown by the framework.

Returns: True if errors are thrown.

Definition at line 272 of file Framework.cpp.

 {
     return _throw_errors;
 }

The documentation for this class was generated from the following files:

tests/framework/Framework.h
tests/framework/Framework.cpp

Public Member Functions

Static Public Member Functions

Detailed Description

Member Function Documentation

◆ add_data_test_case()

◆ add_printer()

◆ add_test_case()

◆ add_test_info()

◆ available_instruments()

◆ clear_test_info()

◆ configure_only()

◆ error_on_missing_assets()

◆ get()

◆ get_profiler()

◆ has_test_info()

◆ init()

◆ log_failed_expectation()

◆ log_info()

◆ log_level()

◆ log_test_end()

◆ log_test_skipped()

◆ log_test_start()

◆ new_fixture_call()

◆ num_iterations()

◆ pop_suite()

◆ print_test_info()

◆ print_test_results()

◆ push_suite()

◆ run()

◆ set_error_on_missing_assets()

◆ set_instruments_info()

◆ set_new_fixture_call()

◆ set_num_iterations()

◆ set_stop_on_error()

◆ set_test_result()

◆ set_throw_errors()

◆ stop_on_error()

◆ test_infos()

◆ throw_errors()