13 #if defined(ARMNN_SERIALIZER) 16 #if defined(ARMNN_TF_LITE_PARSER) 19 #if defined(ARMNN_ONNX_PARSER) 22 #if defined(ARMNN_TFLITE_DELEGATE) 26 #include <tensorflow/lite/builtin_ops.h> 27 #include <tensorflow/lite/c/builtin_op_data.h> 28 #include <tensorflow/lite/c/common.h> 29 #include <tensorflow/lite/optional_debug_tools.h> 30 #include <tensorflow/lite/kernels/builtin_op_kernels.h> 31 #include <tensorflow/lite/interpreter.h> 32 #include <tensorflow/lite/kernels/register.h> 36 #if defined(ARMNN_TFLITE_DELEGATE) 38 const std::shared_ptr<armnn::IRuntime>& runtime =
nullptr)
42 std::unique_ptr<tflite::FlatBufferModel> model = tflite::FlatBufferModel::BuildFromFile(params.
m_ModelPath.c_str());
44 auto tfLiteInterpreter = std::make_unique<Interpreter>();
45 tflite::ops::builtin::BuiltinOpResolver resolver;
47 tflite::InterpreterBuilder builder(*model, resolver);
48 builder(&tfLiteInterpreter);
49 tfLiteInterpreter->AllocateTensors();
56 std::unique_ptr<TfLiteDelegate, decltype(&armnnDelegate::TfLiteArmnnDelegateDelete)>
60 status = tfLiteInterpreter->ModifyGraphWithDelegate(std::move(theArmnnDelegate));
61 if (status == kTfLiteError)
63 ARMNN_LOG(fatal) <<
"Could not register ArmNN TfLite Delegate to TfLiteInterpreter!";
69 std::cout <<
"Running on TfLite without ArmNN delegate\n";
73 std::vector<std::string> inputBindings;
76 inputBindings.push_back(inputName);
83 const size_t numInputs = inputBindings.size();
85 for(
unsigned int inputIndex = 0; inputIndex < numInputs; ++inputIndex)
87 int input = tfLiteInterpreter->inputs()[inputIndex];
88 TfLiteIntArray* inputDims = tfLiteInterpreter->tensor(input)->dims;
91 for (
unsigned int dim = 0; dim < static_cast<unsigned int>(inputDims->size); ++dim)
93 inputSize *= inputDims->data[dim];
96 if (params.
m_InputTypes[inputIndex].compare(
"float") == 0)
98 auto inputData = tfLiteInterpreter->typed_tensor<
float>(input);
100 if(inputData == NULL)
102 ARMNN_LOG(fatal) <<
"Input tensor is null, input type: " 103 "\"" << params.
m_InputTypes[inputIndex] <<
"\" may be incorrect.";
107 std::vector<float> tensorData;
108 PopulateTensorWithDataGeneric<float>(tensorData,
111 [](
const std::string& s)
112 {
return std::stof(s); });
114 std::copy(tensorData.begin(), tensorData.end(), inputData);
116 else if (params.
m_InputTypes[inputIndex].compare(
"qsymms8") == 0)
118 auto inputData = tfLiteInterpreter->typed_tensor<int8_t>(input);
120 if(inputData == NULL)
122 ARMNN_LOG(fatal) <<
"Input tensor is null, input type: " 123 "\"" << params.
m_InputTypes[inputIndex] <<
"\" may be incorrect.";
127 std::vector<int8_t> tensorData;
128 PopulateTensorWithDataGeneric<int8_t>(tensorData,
131 [](
const std::string& s)
134 std::copy(tensorData.begin(), tensorData.end(), inputData);
136 else if (params.
m_InputTypes[inputIndex].compare(
"int") == 0)
138 auto inputData = tfLiteInterpreter->typed_tensor<int32_t>(input);
140 if(inputData == NULL)
142 ARMNN_LOG(fatal) <<
"Input tensor is null, input type: " 143 "\"" << params.
m_InputTypes[inputIndex] <<
"\" may be incorrect.";
147 std::vector<int32_t> tensorData;
148 PopulateTensorWithDataGeneric<int32_t>(tensorData,
151 [](
const std::string& s)
152 {
return std::stoi(s); });
154 std::copy(tensorData.begin(), tensorData.end(), inputData);
156 else if (params.
m_InputTypes[inputIndex].compare(
"qasymm8") == 0)
158 auto inputData = tfLiteInterpreter->typed_tensor<uint8_t>(input);
160 if(inputData == NULL)
162 ARMNN_LOG(fatal) <<
"Input tensor is null, input type: " 163 "\"" << params.
m_InputTypes[inputIndex] <<
"\" may be incorrect.";
167 std::vector<uint8_t> tensorData;
168 PopulateTensorWithDataGeneric<uint8_t>(tensorData,
171 [](
const std::string& s)
174 std::copy(tensorData.begin(), tensorData.end(), inputData);
178 ARMNN_LOG(fatal) <<
"Unsupported input tensor data type \"" << params.
m_InputTypes[inputIndex] <<
"\". ";
186 status = tfLiteInterpreter->Invoke();
189 for (
unsigned int outputIndex = 0; outputIndex < params.
m_OutputNames.size(); ++outputIndex)
191 auto tfLiteDelegateOutputId = tfLiteInterpreter->outputs()[outputIndex];
192 TfLiteIntArray* outputDims = tfLiteInterpreter->tensor(tfLiteDelegateOutputId)->dims;
195 for (
unsigned int dim = 0; dim < static_cast<unsigned int>(outputDims->size); ++dim)
197 outputSize *= outputDims->data[dim];
203 auto tfLiteDelageOutputData = tfLiteInterpreter->typed_tensor<
float>(tfLiteDelegateOutputId);
204 if(tfLiteDelageOutputData == NULL)
206 ARMNN_LOG(fatal) <<
"Output tensor is null, output type: " 207 "\"" << params.
m_OutputTypes[outputIndex] <<
"\" may be incorrect.";
211 for (
int i = 0; i < outputSize; ++i)
213 printf(
"%f ", tfLiteDelageOutputData[i]);
216 else if (params.
m_OutputTypes[outputIndex].compare(
"int") == 0)
218 auto tfLiteDelageOutputData = tfLiteInterpreter->typed_tensor<int32_t>(tfLiteDelegateOutputId);
219 if(tfLiteDelageOutputData == NULL)
221 ARMNN_LOG(fatal) <<
"Output tensor is null, output type: " 222 "\"" << params.
m_OutputTypes[outputIndex] <<
"\" may be incorrect.";
226 for (
int i = 0; i < outputSize; ++i)
228 printf(
"%d ", tfLiteDelageOutputData[i]);
231 else if (params.
m_OutputTypes[outputIndex].compare(
"qsymms8") == 0)
233 auto tfLiteDelageOutputData = tfLiteInterpreter->typed_tensor<int8_t>(tfLiteDelegateOutputId);
234 if(tfLiteDelageOutputData == NULL)
236 ARMNN_LOG(fatal) <<
"Output tensor is null, output type: " 237 "\"" << params.
m_OutputTypes[outputIndex] <<
"\" may be incorrect.";
241 for (
int i = 0; i < outputSize; ++i)
243 printf(
"%d ", tfLiteDelageOutputData[i]);
246 else if (params.
m_OutputTypes[outputIndex].compare(
"qasymm8") == 0)
248 auto tfLiteDelageOutputData = tfLiteInterpreter->typed_tensor<uint8_t>(tfLiteDelegateOutputId);
249 if(tfLiteDelageOutputData == NULL)
251 ARMNN_LOG(fatal) <<
"Output tensor is null, output type: " 252 "\"" << params.
m_OutputTypes[outputIndex] <<
"\" may be incorrect.";
256 for (
int i = 0; i < outputSize; ++i)
258 printf(
"%u ", tfLiteDelageOutputData[i]);
263 ARMNN_LOG(fatal) <<
"Output tensor is null, output type: " 265 "\" may be incorrect. Output type can be specified with -z argument";
268 std::cout << std::endl;
275 template<
typename TParser,
typename TDataType>
277 const std::shared_ptr<armnn::IRuntime>& runtime =
nullptr)
280 mapbox::util::variant<std::vector<float>, std::vector<int>, std::vector<unsigned char>, std::vector<int8_t>>;
282 std::vector<std::vector<TContainer>> inputs;
283 std::vector<std::vector<TContainer>> outputs;
331 armnn::MakeOptional<QuantizationParams>(
337 std::vector<TContainer> inputDataContainers;
338 for(
unsigned int i = 0; i < numInputs; ++i)
342 armnn::MakeOptional<std::string>(
360 inputDataContainers.push_back(tensorData);
362 inputs.push_back(inputDataContainers);
369 std::vector <TContainer> outputDataContainers;
370 for (
unsigned int i = 0; i < numOutputs; ++i)
374 outputDataContainers.push_back(std::vector<float>(model.
GetOutputSize(i)));
377 outputDataContainers.push_back(std::vector<int>(model.
GetOutputSize(i)));
380 outputDataContainers.push_back(std::vector<uint8_t>(model.
GetOutputSize(i)));
383 outputDataContainers.push_back(std::vector<int8_t>(model.
GetOutputSize(i)));
390 outputs.push_back(outputDataContainers);
399 auto inference_duration = model.
Run(inputs[0], outputs[0]);
403 ARMNN_LOG(warning) <<
"The input data was generated, note that the output will not be useful";
408 for (
size_t i = 0; i < numOutputs; i++)
417 mapbox::util::apply_visitor(printer, outputs[0][i]);
420 ARMNN_LOG(info) <<
"\nInference time: " << std::setprecision(2)
421 << std::fixed << inference_duration.count() <<
" ms\n";
426 ARMNN_LOG(info) <<
"Threshold time: " << std::setprecision(2)
428 auto thresholdMinusInference = params.
m_ThresholdTime - inference_duration.count();
429 ARMNN_LOG(info) <<
"Threshold time - Inference time: " << std::setprecision(2)
430 << std::fixed << thresholdMinusInference <<
" ms" <<
"\n";
432 if (thresholdMinusInference < 0)
434 std::string errorMessage =
"Elapsed inference time is greater than provided threshold time.";
444 ARMNN_LOG(info) <<
"Asynchronous Execution... \n";
446 std::chrono::duration<double, std::milli>>>> inferenceResults;
450 std::vector<std::unique_ptr<armnn::experimental::IWorkingMemHandle>> workingMemHandles;
461 inferenceResults.push_back(std::async(
462 std::launch::async, [&model, &workingMemHandleRef, &inputs, &outputs, i]() {
463 return model.
RunAsync(workingMemHandleRef, inputs[i], outputs[i]);
469 for (
unsigned int j = 0; j < inferenceResults.size(); ++j)
472 auto inferenceResult = inferenceResults[j].get();
473 auto inference_duration = std::get<1>(inferenceResult);
474 auto inferenceID = std::get<0>(inferenceResult);
478 ARMNN_LOG(warning) <<
"The input data was generated, note that the output will not be useful";
483 for (
size_t i = 0; i < numOutputs; i++)
494 mapbox::util::apply_visitor(printer, outputs[j][i]);
497 ARMNN_LOG(info) <<
"\nInference time: " << std::setprecision(2)
498 << std::fixed << inference_duration.count() <<
" ms\n";
503 ARMNN_LOG(info) <<
"Threshold time: " << std::setprecision(2)
505 auto thresholdMinusInference = params.
m_ThresholdTime - inference_duration.count();
506 ARMNN_LOG(info) <<
"Threshold time - Inference time: " << std::setprecision(2)
507 << std::fixed << thresholdMinusInference <<
" ms" <<
"\n";
509 if (thresholdMinusInference < 0)
511 ARMNN_LOG(fatal) <<
"Elapsed inference time is greater than provided threshold time. \n";
514 ARMNN_LOG(info) <<
"Asynchronous Execution is finished for Inference ID: " << inferenceID <<
" \n";
537 int main(
int argc,
const char* argv[])
557 if (modelFormat.find(
"armnn") != std::string::npos)
559 #if defined(ARMNN_SERIALIZER) 560 return MainImpl<armnnDeserializer::IDeserializer, float>(ProgramOptions.
m_ExNetParams, runtime);
562 ARMNN_LOG(fatal) <<
"Not built with serialization support.";
566 else if (modelFormat.find(
"onnx") != std::string::npos)
568 #if defined(ARMNN_ONNX_PARSER) 569 return MainImpl<armnnOnnxParser::IOnnxParser, float>(ProgramOptions.
m_ExNetParams, runtime);
571 ARMNN_LOG(fatal) <<
"Not built with Onnx parser support.";
575 else if(modelFormat.find(
"tflite") != std::string::npos)
579 #if defined(ARMNN_TF_LITE_PARSER) 580 return MainImpl<armnnTfLiteParser::ITfLiteParser, float>(ProgramOptions.
m_ExNetParams, runtime);
582 ARMNN_LOG(fatal) <<
"Not built with Tensorflow-Lite parser support.";
591 #if defined(ARMNN_TF_LITE_DELEGATE) 592 return TfLiteDelegateMainImpl(ProgramOptions.
m_ExNetParams, runtime);
594 ARMNN_LOG(fatal) <<
"Not built with Arm NN Tensorflow-Lite delegate support.";
601 ARMNN_LOG(fatal) <<
"Unknown model format: '" << modelFormat
602 <<
"'. Please include 'tflite' or 'onnx'";
ExecuteNetworkParams m_ExNetParams
std::vector< std::string > m_InputTypes
static IRuntimePtr Create(const CreationOptions &options)
std::string m_MLGOTuningFilePath
std::vector< TensorShapePtr > m_InputTensorShapes
QuantizationParams GetInputQuantizationParams(unsigned int inputIndex=0u) const
const std::vector< armnn::BindingPointInfo > & GetOutputBindingInfos() const
void ConfigureLogging(bool printToStandardOutput, bool printToDebugOutput, LogSeverity severity)
Configures the logging behaviour of the ARMNN library.
bool m_EnableFp16TurboMode
std::string m_DynamicBackendsPath
std::string m_DynamicBackendsPath
virtual const char * what() const noexcept override
armnn::IRuntime::CreationOptions m_RuntimeOptions
#define ARMNN_LOG(severity)
bool m_EnableFp16TurboMode
void PopulateTensorWithData(TContainer &tensorData, unsigned int numElements, const std::string &dataTypeStr, const armnn::Optional< QuantizationParams > &qParams, const armnn::Optional< std::string > &dataFile)
std::vector< std::string > m_OutputNames
Copyright (c) 2021 ARM Limited and Contributors.
TfLiteExecutor m_TfLiteExecutor
std::vector< std::string > m_OutputTensorFiles
bool m_VisualizePostOptimizationModel
size_t m_SimultaneousIterations
std::string m_CachedNetworkFilePath
bool m_EnableBf16TurboMode
unsigned int GetOutputSize(unsigned int outputIndex=0u) const
std::vector< std::string > m_InputBindings
std::vector< armnn::BackendId > m_ComputeDevices
std::tuple< armnn::profiling::ProfilingGuid, std::chrono::duration< double, std::milli > > RunAsync(armnn::experimental::IWorkingMemHandle &workingMemHandleRef, const std::vector< TContainer > &inputContainers, std::vector< TContainer > &outputContainers)
std::vector< std::string > m_OutputTypes
std::vector< armnn::TensorShape > m_InputShapes
bool m_GenerateTensorData
Holds all parameters necessary to execute a network Check ExecuteNetworkProgramOptions.cpp for a description of each parameter.
std::vector< std::string > m_OutputBindings
std::vector< armnn::BackendId > m_ComputeDevices
unsigned int m_NumberOfThreads
std::vector< std::string > m_InputNames
bool m_EnableBf16TurboMode
std::vector< std::string > m_InputTensorDataFilePaths
Holds and parses program options for the ExecuteNetwork application.
TfLiteDelegate * TfLiteArmnnDelegateCreate(armnnDelegate::DelegateOptions options)
EmptyOptional is used to initialize the Optional class in case we want to have default value for an O...
bool m_PrintIntermediateLayers
std::string m_CachedNetworkFilePath
std::unique_ptr< armnn::experimental::IWorkingMemHandle > CreateWorkingMemHandle()
mapbox::util::variant< std::vector< float >, std::vector< int >, std::vector< unsigned char >, std::vector< int8_t > > TContainer
std::chrono::duration< double, std::milli > Run(const std::vector< TContainer > &inputContainers, std::vector< TContainer > &outputContainers)
bool m_EnableLayerDetails
int main(int argc, const char *argv[])
Base class for all ArmNN exceptions so that users can filter to just those.
unsigned int GetInputSize(unsigned int inputIndex=0u) const
std::string m_MLGOTuningFilePath
void TfLiteArmnnDelegateDelete(TfLiteDelegate *tfLiteDelegate)
Optional< T > MakeOptional(Args &&... args)
Utility template that constructs an object of type T in-place and wraps it inside an Optional<T> obje...
int MainImpl(const ExecuteNetworkParams ¶ms, const std::shared_ptr< armnn::IRuntime > &runtime=nullptr)
unsigned int m_NumberOfThreads
std::enable_if_t< std::is_unsigned< Source >::value &&std::is_unsigned< Dest >::value, Dest > numeric_cast(Source source)
std::string m_ModelFormat