40 #include <type_traits> 52 Framework::Framework()
53 : _test_filter(nullptr)
57 Instrument::make_instrument<WallClockTimestamps, ScaleFactor::TIME_MS>);
59 Instrument::make_instrument<WallClockTimestamps, ScaleFactor::TIME_S>);
65 Instrument::make_instrument<SchedulerTimestamps, ScaleFactor::TIME_MS>);
67 Instrument::make_instrument<SchedulerTimestamps, ScaleFactor::TIME_S>);
92 Instrument::make_instrument<OpenCLMemoryUsage, ScaleFactor::SCALE_1K>);
94 Instrument::make_instrument<OpenCLMemoryUsage, ScaleFactor::SCALE_1M>);
97 instruments_info = std::make_unique<InstrumentsInfo>();
102 std::set<InstrumentsDescription> types;
104 for(
const auto &instrument : _available_instruments)
106 types.emplace(instrument.first);
112 std::map<TestResult::Status, int> Framework::count_test_results()
const 114 std::map<TestResult::Status, int> counts;
116 for(
const auto &test : _test_results)
118 ++counts[test.second.status];
141 std::string Framework::current_suite_name()
const 143 return join(_test_suite_name.cbegin(), _test_suite_name.cend(),
"/");
148 _test_suite_name.emplace_back(std::move(name));
153 _test_suite_name.pop_back();
158 _test_info.emplace_back(std::move(info));
168 return !_test_info.empty();
173 if(!_test_info.empty())
177 for(
const auto &
str : _test_info)
179 os <<
" " <<
str <<
"\n";
184 template <
typename F>
185 void Framework::func_on_all_printers(F &&func)
194 func_on_all_printers([&](
Printer * p)
203 static_cast<void>(
info);
210 func_on_all_printers([&](
Printer * p)
219 func_on_all_printers([](
Printer * p)
233 if(_log_level >= error.
level())
235 func_on_all_printers([&](
Printer * p)
248 func_on_all_printers([&](
Printer * p)
257 return _num_iterations;
272 return _throw_errors;
282 return _stop_on_error;
292 return _error_on_missing_assets;
309 _current_test_info = &
info;
310 _current_test_result = &result;
314 func_on_all_printers([](
Printer * p)
324 std::unique_ptr<TestCase> test_case = test_factory.
make();
330 test_case->do_setup();
332 for(
int i = 0; i < _num_iterations; ++i)
340 if(_num_iterations == 1 || i != 0)
345 test_case->do_sync();
346 if(_num_iterations == 1 || i != 0)
352 test_case->do_teardown();
365 if(_error_on_missing_assets)
370 func_on_all_printers([&](
Printer * p)
387 func_on_all_printers([&](
Printer * p)
399 if(_log_level >= error.
level())
401 func_on_all_printers([&](
Printer * p)
414 #ifdef ARM_COMPUTE_CL 415 catch(const ::cl::Error &error)
420 std::stringstream stream;
421 stream <<
"Error code: " << error.err();
423 func_on_all_printers([&](
Printer * p)
437 catch(
const std::exception &error)
442 func_on_all_printers([&](
Printer * p)
460 func_on_all_printers([&](
Printer * p)
474 catch(
const std::exception &error)
478 func_on_all_printers([&](
Printer * p)
495 func_on_all_printers([&](
Printer * p)
511 func_on_all_printers([](
Printer * p)
517 _current_test_info =
nullptr;
518 _current_test_result =
nullptr;
532 throw std::runtime_error(
"Abort on first error.");
546 _test_results.clear();
550 func_on_all_printers([](
Printer * p)
556 const std::chrono::time_point<std::chrono::high_resolution_clock> start = std::chrono::high_resolution_clock::now();
561 for(
auto &test_factory : _test_factories)
563 const std::string test_case_name = test_factory->
name();
564 const TestInfo test_info{ id, test_case_name, test_factory->
mode(), test_factory->
status() };
566 if(_test_filter->is_selected(test_info))
568 #ifdef ARM_COMPUTE_CL 575 cl::Context new_ctx = cl::Context(CL_DEVICE_TYPE_DEFAULT, ctx_properties.data());
576 cl::CommandQueue new_queue = cl::CommandQueue(new_ctx,
CLKernelLibrary::get().get_device(), queue_properties);
582 #endif // ARM_COMPUTE_CL 584 run_test(test_info, *test_factory);
595 const std::chrono::time_point<std::chrono::high_resolution_clock>
end = std::chrono::high_resolution_clock::now();
599 func_on_all_printers([](
Printer * p)
605 auto runtime = std::chrono::duration_cast<std::chrono::seconds>(end - start);
606 std::map<TestResult::Status, int> results = count_test_results();
610 std::cout <<
"Executed " << _test_results.size() <<
" test(s) (" 620 return (static_cast<unsigned int>(num_successful_tests) == _test_results.size());
625 _test_results.emplace(std::move(info), std::move(result));
632 for(
const auto &test : _test_results)
647 const bool all_instruments = std::any_of(
648 _instruments.begin(),
655 const auto group =
static_cast<InstrumentType>(
static_cast<uint64_t
>(
type.first) & 0xFF00);
656 return (group == instrument.first) && (instrument.second ==
type.second);
658 != _instruments.end();
661 for(
const auto &instrument : _available_instruments)
663 if(all_instruments ||
is_selected(instrument.first))
665 profiler.
add(instrument.second());
674 _printers.push_back(printer);
679 std::vector<TestInfo> ids;
683 for(
const auto &factory : _test_factories)
685 const TestInfo test_info{ id, factory->
name(), factory->mode(), factory->status() };
687 if(_test_filter->is_selected(test_info))
689 ids.emplace_back(std::move(test_info));
706 *instruments_info = instr_info;
711 return _configure_only;
716 return _new_fixture_call;
721 _new_fixture_call = val;
void pop_suite()
Remove innermost test suite.
virtual std::unique_ptr< TestCase > make() const =0
Factory function to create the test case.
Framework configuration structure.
std::string join(T first, T last, const std::string &separator)
Helper function to concatenate multiple strings.
virtual void print_error(const std::exception &error, bool expected)=0
Print test error.
virtual void print_run_footer()=0
Print footer after running all tests.
bool new_fixture_call() const
Return whether the new fixture has been called.
virtual void print_errors_header()=0
Print header before errors.
void add(std::unique_ptr< Instrument > instrument)
Add instrument to the performance monitor.
int num_iterations
Number of iterations per test.
std::pair< InstrumentType, ScaleFactor > InstrumentsDescription
static CLScheduler & get()
Access the scheduler singleton.
Profiler class to collect benchmark numbers.
void set_context(cl::Context context)
Accessor to set the CL context to be used by the scheduler.
virtual void print_info(const std::string &info)=0
Print test log info.
void print_test_info(std::ostream &os) const
Print test info.
const ActivationSelectorPtr is_selected
void set_throw_errors(bool throw_errors)
Set whether errors are caught or thrown by the framework.
bool stop_on_error() const
Indicates if test execution is stopped after the first failed test.
void log_failed_expectation(const TestError &error)
Tell the framework that the currently running test case failed a non-fatal expectation.
DatasetMode mode() const
Get the mode for which test case will be enabled.
Abstract printer class used by the Framework to present output.
void set_instruments_info(InstrumentsInfo instr_info)
Sets instruments info.
std::unique_ptr< InstrumentsInfo > instruments_info
void log_info(const std::string &info)
Print the debug information that has already been logged.
#define ARM_COMPUTE_ERROR_ON(cond)
If the condition is true then an error message is printed and an exception thrown.
void test_stop()
Call test_stop() on all the added instruments.
std::string header_data
Test header data.
static CLKernelLibrary & get()
Access the KernelLibrary singleton.
bool throw_errors() const
Should errors be caught or thrown by the framework.
void set_error_on_missing_assets(bool error_on_missing_assets)
Set whether a test should be considered as failed if its assets cannot be found.
LogLevel
Severity of the information.
void set_test_result(TestInfo info, TestResult result)
Set the result for an executed test case.
decltype(strategy::transforms) typedef type
bool configure_only() const
Get the configure only flag.
Copyright (c) 2017-2021 Arm Limited.
float cooldown_sec
Delay between tests in seconds.
static Framework & get()
Access to the singleton.
Interface to enqueue OpenCL kernels and get/set the OpenCL CommandQueue and ICLTuner.
Abstract factory class to create test cases.
std::string name
Test name.
std::string name() const
Name of the test case.
LogLevel log_level
Verbosity of the output.
bool run()
Run all enabled test cases.
void clear_programs_cache()
Clear the library's cache of binary programs.
std::vector< framework::InstrumentsDescription > instruments
Instrument types that will be used for benchmarking.
LogLevel level() const
Severity of the error.
cl::Context & context()
Accessor for the associated CL context.
std::vector< TestInfo > test_infos() const
List of TestInfo's.
std::string name_filter
Regular expression to filter tests by name.
std::unique_ptr< ParametersLibrary > parameters
const std::string & header() const
Return JSON formatted header data.
Status status() const
Get the status of the test case.
DatasetMode mode
Dataset mode.
Error class for when some external assets are missing.
std::set< InstrumentsDescription > available_instruments() const
Supported instrument types for benchmarking.
Information about a test case.
void end(TokenStream &in, bool &valid)
LogLevel log_level() const
Get the current logging level.
Profiler get_profiler() const
Factory method to obtain a configured profiler.
cl::CommandQueue & queue()
Accessor for the associated CL command queue.
void log_test_start(const TestInfo &info)
Tell the framework that execution of a test starts.
const MeasurementsMap & measurements() const
Return measurements for all instruments.
void set_queue(cl::CommandQueue queue)
Accessor to set the CL command queue to be used by the scheduler.
void for_each(F &&)
Base case of for_each.
Profiler::MeasurementsMap measurements
Profiling information.
virtual void print_run_header()=0
Print header before running all tests.
ScaleKernelInfo info(interpolation_policy, default_border_mode, PixelValue(), sampling_policy, false)
void log_test_end(const TestInfo &info)
Tell the framework that a test case finished.
void add_test_info(std::string info)
Add info string for the next expectation/assertion.
bool error_on_missing_assets() const
Indicates if a test should be marked as failed when its assets are missing.
virtual void print_test_footer()=0
Print footer after a test.
std::string id_filter
String to match selected test ids.
bool has_test_info() const
Check if any info has been registered.
void set_num_iterations(int num_iterations)
Set number of iterations per test case.
void push_suite(std::string name)
Add a new test suite.
void start()
Call start() on all the added instruments.
virtual void print_profiler_header(const std::string &header_data)=0
Print header data.
Class to store results of a test.
Status status
Execution status.
virtual void print_errors_footer()=0
Print footer after errors.
void sleep_in_seconds(float seconds)
Makes the calling thread to sleep for a specified number of seconds.
bool configure_only
Only configure kernels.
virtual void print_test_header(const TestInfo &info)=0
Print header before a test.
void add_printer(Printer *printer)
Set the printer used for the output of test results.
void stop()
Call stop() on all the added instruments.
TestCaseFactory::Status status
Test status.
void test_start()
Call test_start() on all the added instruments.
void log_test_skipped(const TestInfo &info)
Tell the framework that a test case is skipped.
void set_new_fixture_call(bool val)
Set the new fixture call flag.
Error class for failures during test execution.
void clear_test_info()
Clear the collected test info.
void set_stop_on_error(bool stop_on_error)
Set whether to abort execution after the first failed test.
int num_iterations() const
Number of iterations per test case.
void init(const FrameworkConfig &config)
Init the framework.
virtual void print_measurements(const Profiler::MeasurementsMap &measurements)=0
Print measurements for a test.
void print_test_results(Printer &printer) const
Use the specified printer to output test results from the last run.
bool opencl_is_available()
Check if OpenCL is available.