ArmNN
 24.08
ArmnnPreparedModel.cpp
Go to the documentation of this file.
1 //
2 // Copyright © 2022 Arm Ltd and Contributors. All rights reserved.
3 // SPDX-License-Identifier: MIT
4 //
5 
6 #define LOG_TAG "arm-armnn-sl"
7 
8 #include "ArmnnPreparedModel.hpp"
9 #include "CanonicalUtils.hpp"
10 
11 #include <DefaultExecution.h>
12 #include <LegacyUtils.h>
13 #include <nnapi/IBurst.h>
14 #include <nnapi/IPreparedModel.h>
15 #include <nnapi/Result.h>
16 #include <nnapi/SharedMemory.h>
17 #include <nnapi/TypeUtils.h>
18 #include <nnapi/Types.h>
19 #include <nnapi/Validation.h>
20 
21 #include <memory>
22 #include <tuple>
23 #include <utility>
24 #include <vector>
25 
26 using namespace android;
27 using namespace android::nn;
28 
29 static const Timing g_NoTiming = {};
30 
31 namespace {
32 
33 using namespace armnn_driver;
34 
35 unsigned long MicrosecondsDuration(android::nn::TimePoint endPoint, android::nn::TimePoint startPoint)
36 {
37  return static_cast<unsigned long>(std::chrono::duration_cast<std::chrono::microseconds>(
38  endPoint - startPoint).count());
39 }
40 
41 bool ValidateRequestArgument(const Request::Argument& requestArg, const armnn::TensorInfo& tensorInfo)
42 {
43  if (requestArg.dimensions.size() != 0)
44  {
45  if (requestArg.dimensions.size() != tensorInfo.GetNumDimensions())
46  {
47  VLOG(DRIVER) << "Mismatched dimensions (request argument: "
48  << requestArg.dimensions.size() << " expected: " << tensorInfo.GetNumDimensions();
49  return false;
50  }
51 
52  for (unsigned int d = 0; d < tensorInfo.GetNumDimensions(); ++d)
53  {
54  if (requestArg.dimensions[d] != 0 && requestArg.dimensions[d] != tensorInfo.GetShape()[d])
55  {
56  VLOG(DRIVER) << "Mismatched dimensions " << d
57  << " (request argument: " << requestArg.dimensions[d]
58  << " expected: " << tensorInfo.GetShape()[d];
59  return false;
60  }
61  }
62  }
63 
64  return true;
65 }
66 
67 armnn::Tensor GetTensorForRequestArgument(const Request::Argument& requestArg,
68  const armnn::TensorInfo& tensorInfo,
69  const std::vector<::android::nn::RunTimePoolInfo>& requestPools)
70 {
71  if (!ValidateRequestArgument(requestArg, tensorInfo))
72  {
73  return armnn::Tensor();
74  }
75 
76  if (requestArg.lifetime == Request::Argument::LifeTime::POINTER)
77  {
78  return armnn::Tensor(tensorInfo, GetMemoryFromPointer(requestArg));
79  }
80  else if (requestArg.lifetime == Request::Argument::LifeTime::POOL)
81  {
82  return armnn::Tensor(tensorInfo, GetMemoryFromPool(requestArg.location, requestPools));
83  }
84  return armnn::Tensor();
85 }
86 
87 inline std::string BuildTensorName(const char* tensorNamePrefix, std::size_t index)
88 {
89  return tensorNamePrefix + std::to_string(index);
90 }
91 
92 bool IsPointerTypeMemory(const Request& request)
93 {
94  for (auto& input : request.inputs)
95  {
96  if (input.lifetime != Request::Argument::LifeTime::POINTER)
97  {
98  return false;
99  }
100  }
101 
102  for (auto& output: request.outputs)
103  {
104  if (output.lifetime != Request::Argument::LifeTime::POINTER)
105  {
106  return false;
107  }
108  }
109 
110  return true;
111 }
112 
113 } // anonymous namespace
114 
115 using namespace android::nn;
116 
117 namespace armnn_driver
118 {
119 
120 void ArmnnPreparedModel::Init()
121 {
122  // Enable profiling if required.
123  m_Runtime->GetProfiler(m_NetworkId)->EnableProfiling(m_GpuProfilingEnabled);
124 }
125 
126 ArmnnPreparedModel::ArmnnPreparedModel(armnn::NetworkId networkId,
127  armnn::IRuntime* runtime,
128  const Model& model,
129  const std::string& requestInputsAndOutputsDumpDir,
130  const bool gpuProfilingEnabled,
131  Priority priority)
132  : m_NetworkId(networkId)
133  , m_Runtime(runtime)
134  , m_Model(model)
135  , m_RequestInputsAndOutputsDumpDir(requestInputsAndOutputsDumpDir)
136  , m_GpuProfilingEnabled(gpuProfilingEnabled)
137  , m_ModelPriority(priority)
138  , m_PrepareFromCache(false)
139 {
140  Init();
141 }
142 
144  armnn::IRuntime* runtime,
145  const std::string& requestInputsAndOutputsDumpDir,
146  const bool gpuProfilingEnabled,
147  Priority priority,
148  const bool prepareModelFromCache)
149  : m_NetworkId(networkId)
150  , m_Runtime(runtime)
151  , m_RequestInputsAndOutputsDumpDir(requestInputsAndOutputsDumpDir)
152  , m_GpuProfilingEnabled(gpuProfilingEnabled)
153  , m_ModelPriority(priority)
154  , m_PrepareFromCache(prepareModelFromCache)
155 {
156  Init();
157 }
158 
159 
160 ErrorStatus ArmnnPreparedModel::PrepareMemoryForInputs(
161  armnn::InputTensors& inputs,
162  const Request& request,
163  const std::vector<android::nn::RunTimePoolInfo>& memPools) const
164 {
165  inputs.reserve(request.inputs.size());
166  for (unsigned int i = 0; i < request.inputs.size(); i++)
167  {
168  const auto& inputArg = request.inputs[i];
169 
170  armnn::TensorInfo inputTensorInfo = m_Runtime->GetInputTensorInfo(m_NetworkId, i);
171  // inputs (of type InputTensors) is composed of a vector of ConstTensors.
172  // Therefore, set all TensorInfo isConstant parameters of input Tensors to true.
173  inputTensorInfo.SetConstant();
174  const armnn::Tensor inputTensor = GetTensorForRequestArgument(inputArg, inputTensorInfo, memPools);
175 
176  if (inputTensor.GetMemoryArea() == nullptr)
177  {
178  VLOG(DRIVER) << "Cannot execute request. Error converting request input " << i << "to tensor.";
179  return ErrorStatus::GENERAL_FAILURE;
180  }
181  inputs.emplace_back(i, inputTensor);
182  }
183 
184  return ErrorStatus::NONE;
185 }
186 
187 ErrorStatus ArmnnPreparedModel::PrepareMemoryForOutputs(
188  armnn::OutputTensors& outputs,
189  std::vector<OutputShape> &outputShapes,
190  const Request& request,
191  const std::vector<android::nn::RunTimePoolInfo>& memPools) const
192 {
193  outputs.reserve(request.outputs.size());
194  for (unsigned int i = 0; i < request.outputs.size(); i++)
195  {
196  auto& outputArg = request.outputs[i];
197 
198  armnn::TensorInfo outputTensorInfo = m_Runtime->GetOutputTensorInfo(m_NetworkId, i);
199  armnn::Tensor outputTensor = GetTensorForRequestArgument(outputArg, outputTensorInfo, memPools);
200  if (outputTensor.GetMemoryArea() == nullptr)
201  {
202  VLOG(DRIVER) << "Cannot execute request. Error converting request output " << i << "to tensor.";
203  return ErrorStatus::GENERAL_FAILURE;
204  }
205 
206  const size_t outputSize = outputTensorInfo.GetNumBytes();
207 
208  unsigned int count = 0;
209  std::for_each(outputArg.dimensions.begin(), outputArg.dimensions.end(), [&](auto dim)
210  {
211  if (dim != 0)
212  {
213  outputTensorInfo.GetShape()[count] = dim;
214  }
215  else
216  {
217  outputTensorInfo.GetShape()[count] = outputArg.dimensions.size();
218  }
219 
220  count++;
221  });
222 
223  outputs.emplace_back(i, outputTensor);
224  outputShapes[i] = ComputeShape(outputTensorInfo);
225 
226  if (outputArg.location.length < outputSize)
227  {
228  VLOG(DRIVER) << "ArmnnPreparedModel::Execute failed outputArg.location.length "
229  << std::to_string(outputArg.location.length).c_str()
230  << " < outputSize " << std::to_string(outputSize).c_str();
231  outputShapes[i].isSufficient = false;
232  return ErrorStatus::OUTPUT_INSUFFICIENT_SIZE;
233  }
234 
235  if (outputArg.lifetime == Request::Argument::LifeTime::POOL)
236  {
237  size_t bufferSize = memPools.at(outputArg.location.poolIndex).getSize();
238  if (bufferSize < outputSize)
239  {
240  VLOG(DRIVER) << "ArmnnPreparedModel::Execute failed bufferSize "
241  << std::to_string(outputArg.location.length).c_str()
242  << " < outputSize " << std::to_string(outputSize).c_str();
243  outputShapes[i].isSufficient = false;
244  return ErrorStatus::OUTPUT_INSUFFICIENT_SIZE;
245  }
246  }
247  }
248  return ErrorStatus::NONE;
249 }
250 
251 ErrorStatus ArmnnPreparedModel::PrepareMemoryForIO(armnn::InputTensors& inputs,
252  armnn::OutputTensors& outputs,
253  std::vector<android::nn::RunTimePoolInfo>& memPools,
254  const Request& request,
255  const bool pointerMemory) const
256 {
257  //Check memory pools are not empty
258  // add the inputs and outputs with their data
259  try
260  {
261  if (!pointerMemory && !setRunTimePoolInfosFromMemoryPools(&memPools, request.pools))
262  {
263  return ErrorStatus::INVALID_ARGUMENT;
264  }
265 
266  if (PrepareMemoryForInputs(inputs, request, memPools) != ErrorStatus::NONE)
267  {
268  VLOG(DRIVER) << "Failed when preparing memory for Inputs";
269  return ErrorStatus::GENERAL_FAILURE;
270  }
271 
272  std::vector<OutputShape> outputShapes(request.outputs.size());
273 
274  auto errorStatus = PrepareMemoryForOutputs(outputs, outputShapes, request, memPools);
275  if (errorStatus != ErrorStatus::NONE)
276  {
277  return errorStatus;
278  }
279  }
280  catch (armnn::Exception& e)
281  {
282  VLOG(DRIVER) << "armnn::Exception caught while preparing for EnqueueWorkload: " << e.what();
283  return ErrorStatus::GENERAL_FAILURE;
284  }
285  catch (std::exception& e)
286  {
287  VLOG(DRIVER) << "std::exception caught while preparing for EnqueueWorkload: " << e.what();
288  return ErrorStatus::GENERAL_FAILURE;
289  }
290 
291  return ErrorStatus::NONE;
292 }
293 
294 ExecutionResult<std::pair<std::vector<OutputShape>, Timing>> ArmnnPreparedModel::execute(
295  const Request& request,
296  MeasureTiming measureTiming,
297  const OptionalTimePoint& deadline,
298  const OptionalDuration&,
299  const std::vector<android::nn::TokenValuePair>& hints,
300  const std::vector<android::nn::ExtensionNameAndPrefix>& extensionNameToPrefix) const
301 {
302  VLOG(DRIVER) << "CanonicalDriver::PreparedModel::execute()";
303 
305  if (measureTiming == MeasureTiming::YES)
306  {
307  ctx.measureTimings = measureTiming;
308  ctx.driverStart = Clock::now();
309  }
310 
311  if (!m_PrepareFromCache)
312  {
313  const auto modelRequest = validateRequestForModel(request, m_Model);
314  if (!modelRequest.ok())
315  {
316  return NN_ERROR(ErrorStatus::INVALID_ARGUMENT) << modelRequest.error();
317  }
318  VLOG(DRIVER) << "ArmnnPreparedModel::execute(): " << GetModelSummary(m_Model).c_str();
319  }
320  if (hasDeadlinePassed(deadline))
321  {
322  return NN_ERROR(ErrorStatus::MISSED_DEADLINE_PERSISTENT);
323  }
324 
325  // map the memory pool into shared pointers
326  // use a shared memory pools vector on the heap, as it is passed to the request thread
327  auto memPools = std::make_shared<std::vector<android::nn::RunTimePoolInfo>>();
328 
329  // allocate the tensors on the heap, as they are passed to the request thread
330  auto inputTensors = std::make_shared<armnn::InputTensors>();
331  auto outputTensors = std::make_shared<armnn::OutputTensors>();
332 
333  auto isPointerTypeMemory = IsPointerTypeMemory(request);
334  ErrorStatus theErrorStatus = PrepareMemoryForIO(*inputTensors,
335  *outputTensors,
336  *memPools,
337  request,
338  isPointerTypeMemory);
339 
340  switch(theErrorStatus)
341  {
342  case ErrorStatus::OUTPUT_INSUFFICIENT_SIZE:
343  return NN_ERROR(ErrorStatus::OUTPUT_INSUFFICIENT_SIZE);
344  case ErrorStatus::GENERAL_FAILURE:
345  return NN_ERROR(ErrorStatus::GENERAL_FAILURE);
346  case ErrorStatus::INVALID_ARGUMENT:
347  return NN_ERROR(ErrorStatus::INVALID_ARGUMENT);
348  default:
349  {}
350  }
351 
352  std::vector<OutputShape> outputShapes(outputTensors->size());
353  for (unsigned int i = 0; i < outputTensors->size(); i++)
354  {
355  std::pair<int, armnn::Tensor> outputTensorPair = (*outputTensors)[i];
356  const armnn::Tensor outputTensor = outputTensorPair.second;
357  const armnn::TensorInfo outputTensorInfo = outputTensor.GetInfo();
358 
359  outputShapes[i] = ComputeShape(outputTensorInfo);
360  }
361  Timing theTiming;
362 
363  VLOG(DRIVER) << "ArmnnPreparedModel::execute(...) before ExecuteGraph";
364  auto errorStatus = ExecuteGraph(memPools, *inputTensors, *outputTensors, ctx, isPointerTypeMemory);
365  if (errorStatus != ErrorStatus::NONE)
366  {
367  return NN_ERROR(errorStatus) << "execute() failed";
368  }
369  VLOG(DRIVER) << "ArmnnPreparedModel::execute(...) after ExecuteGraph";
370 
371  return std::make_pair(outputShapes, theTiming);
372 }
373 
375  std::shared_ptr<std::vector<android::nn::RunTimePoolInfo>>& pMemPools,
376  armnn::InputTensors& inputTensors,
377  armnn::OutputTensors& outputTensors,
379  const bool pointerMemory) const
380 {
381  VLOG(DRIVER) << "ArmnnPreparedModel::ExecuteGraph(...)";
382 
383  DumpTensorsIfRequired("Input", inputTensors);
384  std::vector<armnn::ImportedInputId> importedInputIds;
385  std::vector<armnn::ImportedOutputId> importedOutputIds;
386  try
387  {
388  if (ctx.measureTimings == MeasureTiming::YES)
389  {
390  ctx.deviceStart = Clock::now();
391  }
392  armnn::Status status;
393  VLOG(DRIVER) << "ArmnnPreparedModel::ExecuteGraph m_AsyncModelExecutionEnabled false";
394  importedInputIds = m_Runtime->ImportInputs(m_NetworkId, inputTensors, armnn::MemorySource::Malloc);
395  if (!importedInputIds.empty())
396  {
397  // Some or all of the input tensors been imported. We need to remove the ones that could from
398  // inputTensors.
399  for (armnn::ImportedInputId& importedId : importedInputIds)
400  {
401  inputTensors.erase(
402  std::remove_if(
403  inputTensors.begin(), inputTensors.end(),
404  [&importedId](std::pair<armnn::LayerBindingId, class armnn::ConstTensor>& element) {
405  return (element.first == static_cast<int>(importedId));
406  }),
407  inputTensors.end());
408  }
409  }
410  importedOutputIds = m_Runtime->ImportOutputs(m_NetworkId, outputTensors, armnn::MemorySource::Malloc);
411  if (!importedOutputIds.empty())
412  {
413  // Some or all of the output tensors could not be imported. We need to remove the ones that could
414  // from outputTensors.
415  for (armnn::ImportedInputId& importedId : importedOutputIds)
416  {
417  outputTensors.erase(
418  std::remove_if(
419  outputTensors.begin(), outputTensors.end(),
420  [&importedId](std::pair<armnn::LayerBindingId, class armnn::Tensor>& element) {
421  return (element.first == static_cast<int>(importedId));
422  }),
423  outputTensors.end());
424  }
425  }
426  status = m_Runtime->EnqueueWorkload(m_NetworkId,
427  inputTensors,
428  outputTensors,
429  importedInputIds,
430  importedOutputIds);
431 
432  if (ctx.measureTimings == MeasureTiming::YES)
433  {
434  ctx.deviceEnd = Clock::now();
435  }
436  if (status != armnn::Status::Success)
437  {
438  VLOG(DRIVER) << "ArmnnPreparedModel:ExecuteGraph EnqueueWorkload failed";
439  return ErrorStatus::GENERAL_FAILURE;
440  }
441  }
442  catch (armnn::Exception& e)
443  {
444  VLOG(DRIVER) << "armnn:Exception caught from EnqueueWorkload: " << e.what();
445  return ErrorStatus::GENERAL_FAILURE;
446  }
447  catch (std::exception& e)
448  {
449  VLOG(DRIVER) << "std::exception caught from EnqueueWorkload: " << e.what();
450  return ErrorStatus::GENERAL_FAILURE;
451  }
452 
453  if (!pointerMemory && (!importedInputIds.empty() || !importedOutputIds.empty()))
454  {
455  CommitPools(*pMemPools);
456  }
457  DumpTensorsIfRequired("Output", outputTensors);
458 
459  if (ctx.measureTimings == MeasureTiming::YES)
460  {
461  ctx.driverEnd = Clock::now();
462  Timing timing;
463  timing.timeOnDevice = ctx.deviceEnd - ctx.deviceStart;
464  timing.timeInDriver = ctx.driverEnd - ctx.driverStart;
465  VLOG(DRIVER) << "ArmnnPreparedModel::execute timing - Device = "
466  << timing.timeOnDevice << "Driver = " << timing.timeInDriver;
467  }
468  return ErrorStatus::NONE;
469 }
470 
472 {
473  return m_ModelPriority;
474 }
475 
476 
477 GeneralResult<std::pair<SyncFence, ExecuteFencedInfoCallback>> ArmnnPreparedModel::executeFenced(
478  const Request& request,
479  const std::vector<SyncFence>& waitFor,
480  MeasureTiming measureTiming,
481  const OptionalTimePoint& deadline,
482  const OptionalDuration&,
483  const OptionalDuration&,
484  const std::vector<android::nn::TokenValuePair>& hints,
485  const std::vector<android::nn::ExtensionNameAndPrefix>& extensionNameToPrefix) const
486 {
487  VLOG(DRIVER) << "ArmnnPreparedModel::executeFenced()";
488 
489  if (!m_PrepareFromCache) {
490  const auto modelRequest = validateRequestForModel(request, m_Model);
491  if (!modelRequest.ok())
492  {
493  return NN_ERROR(ErrorStatus::INVALID_ARGUMENT) << modelRequest.error();
494  }
495  VLOG(DRIVER) << "ArmnnPreparedModel::executeFenced(): " << GetModelSummary(m_Model).c_str();
496  }
497  if (hasDeadlinePassed(deadline))
498  {
499  return NN_ERROR(ErrorStatus::MISSED_DEADLINE_PERSISTENT);
500  }
501 
503  if (measureTiming == MeasureTiming::YES)
504  {
505  ctx.measureTimings = measureTiming;
506  ctx.driverStart = Clock::now();
507  }
508 
509  // Wait for the dependent events to signal
510  for (const auto& syncFence : waitFor)
511  {
512  if (!syncFence.getSharedHandle())
513  {
514  return NN_ERROR(ErrorStatus::INVALID_ARGUMENT);
515  }
516  if (syncFence.syncWait({}) != SyncFence::FenceState::SIGNALED)
517  {
518  return NN_ERROR(ErrorStatus::GENERAL_FAILURE) << "syncWait failed";
519  }
520  }
521 
522  android::nn::TimePoint fenceExecutionStart;
523  if (measureTiming == MeasureTiming::YES)
524  {
525  fenceExecutionStart = Clock::now();
526  }
527 
528  // map the memory pool into shared pointers
529  // use a shared memory pools vector on the heap, as it is passed to the request thread
530  auto memPools = std::make_shared<std::vector<android::nn::RunTimePoolInfo>>();
531 
532  // allocate the tensors on the heap, as they are passed to the request thread
533  auto inputTensors = std::make_shared<armnn::InputTensors>();
534  auto outputTensors = std::make_shared<armnn::OutputTensors>();
535 
536  auto isPointerTypeMemory = IsPointerTypeMemory(request);
537  ErrorStatus theErrorStatus = PrepareMemoryForIO(*inputTensors,
538  *outputTensors,
539  *memPools,
540  request,
541  isPointerTypeMemory);
542 
543  if (theErrorStatus != ErrorStatus::NONE)
544  {
545  return NN_ERROR(ErrorStatus::INVALID_ARGUMENT) << "executeFenced() failed";
546  }
547 
548  Timing timingSinceLaunch = {};
549  Timing timingAfterFence = {};
550  if (measureTiming == MeasureTiming::YES)
551  {
552  timingAfterFence.timeOnDevice = ctx.deviceEnd - ctx.deviceStart;
553  timingAfterFence.timeInDriver = ctx.driverEnd - fenceExecutionStart;
554  VLOG(DRIVER) << "executeFenced timingSinceLaunch = " << timingAfterFence.timeOnDevice;
555  VLOG(DRIVER) << "executeFenced timingAfterFence = " << timingAfterFence.timeInDriver;
556  }
557 
558  VLOG(DRIVER) << "ArmnnCanonicalPreparedModel::executeFenced(...) before ExecuteGraph";
559  auto errorStatus = ExecuteGraph(memPools, *inputTensors, *outputTensors, ctx, isPointerTypeMemory);
560  VLOG(DRIVER) << "ArmnnCanonicalPreparedModel::executeFenced(...) after ExecuteGraph";
561 
562  ExecuteFencedInfoCallback armnnFencedExecutionCallback =
563  [timingSinceLaunch, timingAfterFence, errorStatus]() {
564 
565  GeneralResult<std::pair<Timing, Timing>> result;
566 
567  switch(errorStatus)
568  {
569  case ErrorStatus::OUTPUT_INSUFFICIENT_SIZE:
570  result.error().code = (ErrorStatus::OUTPUT_INSUFFICIENT_SIZE);
571  case ErrorStatus::GENERAL_FAILURE:
572  result.error().code = (ErrorStatus::GENERAL_FAILURE);
573  case ErrorStatus::INVALID_ARGUMENT:
574  result.error().code = (ErrorStatus::INVALID_ARGUMENT);
575  default:
576  {
577  result.value() = std::make_pair(timingSinceLaunch, timingAfterFence);
578  }
579  }
580  return result;
581  };
582  return std::make_pair(SyncFence::createAsSignaled(), std::move(armnnFencedExecutionCallback ));
583 }
584 
585 GeneralResult<SharedExecution> ArmnnPreparedModel::createReusableExecution(
586  const Request& request,
587  MeasureTiming measureTiming,
588  const OptionalDuration& loopTimeoutDuration,
589  const std::vector<android::nn::TokenValuePair>& hints,
590  const std::vector<android::nn::ExtensionNameAndPrefix>& extensionNameToPrefix) const
591 {
592  VLOG(DRIVER) << "ArmnnPreparedModel::createReusableExecution()";
593  return std::make_shared<DefaultExecution>(shared_from_this(),
594  request,
595  measureTiming,
596  loopTimeoutDuration);
597 }
598 
599 GeneralResult<SharedBurst> ArmnnPreparedModel::configureExecutionBurst() const
600 {
601  return nullptr;
602 }
603 
605 {
606  return &m_Model;
607 }
608 
609 template<typename TensorBindingCollection>
610 void ArmnnPreparedModel::DumpTensorsIfRequired(char const* tensorNamePrefix,
611  const TensorBindingCollection& tensorBindings) const
612 {
613  if (!m_RequestInputsAndOutputsDumpDir.empty())
614  {
615  const std::string requestName = std::to_string(m_NetworkId) + ".dump";
616  for (std::size_t i = 0u; i < tensorBindings.size(); ++i)
617  {
618  DumpTensor(m_RequestInputsAndOutputsDumpDir,
619  requestName,
620  BuildTensorName(tensorNamePrefix, i),
621  tensorBindings[i].second);
622  }
623  }
624 }
625 
627 {
628  VLOG(DRIVER) << "ArmnnPreparedModel::~ArmnnPreparedModel()";
629  // Get a hold of the profiler used by this model.
630  if (m_GpuProfilingEnabled)
631  {
632  auto profiler = m_Runtime->GetProfiler(m_NetworkId);
633  if (profiler)
634  {
635  // Dump the profiling info to a file if required.
636  DumpJsonProfilingIfRequired(m_GpuProfilingEnabled,
637  m_RequestInputsAndOutputsDumpDir,
638  m_NetworkId,
639  profiler.get());
640  }
641  }
642  // Unload the network associated with this model
643  m_Runtime->UnloadNetwork(m_NetworkId);
644 }
645 
646 bool ArmnnPreparedModel::ExecuteWithDummyInputs(unsigned int numInputs, unsigned int numOutputs) const
647 {
648  std::vector<std::vector<char>> storage;
649  armnn::InputTensors inputTensors;
650  for (unsigned int i = 0; i < numInputs; i++)
651  {
652  armnn::TensorInfo inputTensorInfo = m_Runtime->GetInputTensorInfo(m_NetworkId, i);
653  // pInputTensors (of type InputTensors) is composed of a vector of ConstTensors.
654  // Therefore, set all TensorInfo isConstant parameters of input Tensors to true.
655  inputTensorInfo.SetConstant();
656  storage.emplace_back(inputTensorInfo.GetNumBytes());
657  const armnn::ConstTensor inputTensor(inputTensorInfo, storage.back().data());
658 
659  inputTensors.emplace_back(i, inputTensor);
660  }
661 
662  armnn::OutputTensors outputTensors;
663  for (unsigned int i = 0; i < numOutputs; i++)
664  {
665  const armnn::TensorInfo outputTensorInfo = m_Runtime->GetOutputTensorInfo(m_NetworkId, i);
666  storage.emplace_back(outputTensorInfo.GetNumBytes());
667  const armnn::Tensor outputTensor(outputTensorInfo, storage.back().data());
668 
669  outputTensors.emplace_back(i, outputTensor);
670  }
672  ctx.measureTimings = MeasureTiming::NO;
673  auto memPools = std::make_shared<std::vector<::android::nn::RunTimePoolInfo>>();
674 
675  auto errorStatus = ExecuteGraph(memPools,
676  inputTensors,
677  outputTensors,
678  ctx);
679 
680  return errorStatus == ErrorStatus::NONE;
681 }
682 
683 } // namespace armnn_driver
armnn::MemorySource::Malloc
@ Malloc
armnn::ImportedInputId
unsigned int ImportedInputId
Definition: Types.hpp:310
armnn::Tensor
A tensor defined by a TensorInfo (shape and data type) and a mutable backing store.
Definition: Tensor.hpp:321
armnn::BaseTensor::GetMemoryArea
MemoryType GetMemoryArea() const
Definition: Tensor.hpp:307
armnn::TensorInfo::GetNumBytes
unsigned int GetNumBytes() const
Definition: Tensor.cpp:427
armnn_driver::DumpJsonProfilingIfRequired
void DumpJsonProfilingIfRequired(bool gpuProfilingEnabled, const std::string &dumpDir, armnn::NetworkId networkId, const armnn::IProfiler *profiler)
Definition: CanonicalUtils.cpp:352
armnn_driver::ArmnnPreparedModel::GetModelPriority
Priority GetModelPriority() const
Definition: ArmnnPreparedModel.cpp:471
armnn::InputTensors
std::vector< std::pair< LayerBindingId, class ConstTensor > > InputTensors
Definition: Tensor.hpp:394
armnn_driver::CanonicalExecutionContext::deviceEnd
android::nn::TimePoint deviceEnd
Definition: ArmnnPreparedModel.hpp:36
armnn_driver::CommitPools
void CommitPools(std::vector<::android::nn::RunTimePoolInfo > &memPools)
Definition: CanonicalUtils.cpp:615
armnn::TensorInfo
Definition: Tensor.hpp:152
armnn::TensorInfo::GetNumDimensions
unsigned int GetNumDimensions() const
Definition: Tensor.hpp:197
armnn::OutputTensors
std::vector< std::pair< LayerBindingId, class Tensor > > OutputTensors
Definition: Tensor.hpp:395
armnn_driver::ArmnnPreparedModel::ArmnnPreparedModel
ArmnnPreparedModel(armnn::NetworkId networkId, armnn::IRuntime *runtime, const Model &model, const std::string &requestInputsAndOutputsDumpDir, const bool gpuProfilingEnabled, Priority priority=Priority::MEDIUM)
Definition: ArmnnPreparedModel.cpp:126
armnn::IRuntime
Definition: IRuntime.hpp:75
armnn::Exception::what
virtual const char * what() const noexcept override
Definition: Exceptions.cpp:32
armnn::IRuntime::ImportInputs
std::vector< ImportedInputId > ImportInputs(NetworkId networkId, const InputTensors &inputTensors, MemorySource forceImportMemorySource=MemorySource::Undefined)
ImportInputs separates the importing and mapping of InputTensors from network execution.
Definition: Runtime.cpp:92
ArmnnPreparedModel.hpp
CanonicalUtils.hpp
armnn::NetworkId
int NetworkId
Definition: IRuntime.hpp:35
armnn_driver::ComputeShape
OutputShape ComputeShape(const armnn::TensorInfo &info)
Definition: CanonicalUtils.hpp:95
armnn_driver::ArmnnPreparedModel::getUnderlyingResource
std::any getUnderlyingResource() const override
Definition: ArmnnPreparedModel.cpp:604
armnn_driver::ErrorStatus
::android::nn::ErrorStatus ErrorStatus
Definition: ConversionUtils.hpp:51
armnn_driver::CanonicalExecutionContext::driverEnd
android::nn::TimePoint driverEnd
Definition: ArmnnPreparedModel.hpp:34
armnn_driver
Helper classes.
Definition: ArmnnDevice.cpp:37
armnn_driver::GetMemoryFromPool
void * GetMemoryFromPool(DataLocation location, const std::vector< android::nn::RunTimePoolInfo > &memPools)
Returns a pointer to a specific location in a pool`.
Definition: CanonicalUtils.cpp:66
armnn_driver::Model
::android::nn::Model Model
Helper classes.
Definition: ConversionUtils.hpp:45
armnn_driver::ArmnnPreparedModel::ExecuteWithDummyInputs
bool ExecuteWithDummyInputs(unsigned int numInputs, unsigned int numOutputs) const
Executes this model with dummy inputs (e.g.
Definition: ArmnnPreparedModel.cpp:646
armnn_driver::CanonicalExecutionContext
Definition: ArmnnPreparedModel.hpp:29
armnn::IRuntime::GetOutputTensorInfo
TensorInfo GetOutputTensorInfo(NetworkId networkId, LayerBindingId layerId) const
Definition: Runtime.cpp:87
armnn::Status::Success
@ Success
armnn::Exception
Base class for all ArmNN exceptions so that users can filter to just those.
Definition: Exceptions.hpp:46
armnn_driver::GetModelSummary
std::string GetModelSummary(const Model &model)
Definition: CanonicalUtils.cpp:526
armnn_driver::ArmnnPreparedModel::~ArmnnPreparedModel
virtual ~ArmnnPreparedModel()
Definition: ArmnnPreparedModel.cpp:626
armnn::BaseTensor::GetInfo
const TensorInfo & GetInfo() const
Definition: Tensor.hpp:297
armnn::IRuntime::GetProfiler
const std::shared_ptr< IProfiler > GetProfiler(NetworkId networkId) const
Gets the profiler corresponding to the given network id.
Definition: Runtime.cpp:151
armnn_driver::CanonicalExecutionContext::driverStart
android::nn::TimePoint driverStart
Definition: ArmnnPreparedModel.hpp:33
armnn_driver::CanonicalExecutionContext::deviceStart
android::nn::TimePoint deviceStart
Definition: ArmnnPreparedModel.hpp:35
armnn_driver::CanonicalExecutionContext::measureTimings
::android::nn::MeasureTiming measureTimings
Definition: ArmnnPreparedModel.hpp:31
armnn::IRuntime::UnloadNetwork
Status UnloadNetwork(NetworkId networkId)
Unloads a network from the IRuntime.
Definition: Runtime.cpp:136
armnn::Status
Status
Definition: Types.hpp:42
armnn_driver::ArmnnPreparedModel::execute
ExecutionResult< std::pair< std::vector< OutputShape >, Timing > > execute(const Request &request, MeasureTiming measureTiming, const OptionalTimePoint &deadline, const OptionalDuration &loopTimeoutDuration, const std::vector< android::nn::TokenValuePair > &hints, const std::vector< android::nn::ExtensionNameAndPrefix > &extensionNameToPrefix) const override
Definition: ArmnnPreparedModel.cpp:294
armnn_driver::GetMemoryFromPointer
void * GetMemoryFromPointer(const Request::Argument &requestArg)
Definition: CanonicalUtils.cpp:77
armnn::TensorInfo::GetShape
const TensorShape & GetShape() const
Definition: Tensor.hpp:193
android::nn
Definition: support_library_service.cpp:10
armnn_driver::ArmnnPreparedModel::executeFenced
GeneralResult< std::pair< SyncFence, ExecuteFencedInfoCallback > > executeFenced(const Request &request, const std::vector< SyncFence > &waitFor, MeasureTiming measureTiming, const OptionalTimePoint &deadline, const OptionalDuration &loopTimeoutDuration, const OptionalDuration &timeoutDurationAfterFence, const std::vector< android::nn::TokenValuePair > &hints, const std::vector< android::nn::ExtensionNameAndPrefix > &extensionNameToPrefix) const override
Definition: ArmnnPreparedModel.cpp:477
armnn_driver::DumpTensor
void DumpTensor(const std::string &dumpDir, const std::string &requestName, const std::string &tensorName, const TensorType &tensor)
Definition: CanonicalUtils.cpp:219
armnn_driver::ArmnnPreparedModel::createReusableExecution
GeneralResult< android::nn::SharedExecution > createReusableExecution(const Request &request, MeasureTiming measureTiming, const OptionalDuration &loopTimeoutDuration, const std::vector< android::nn::TokenValuePair > &hints, const std::vector< android::nn::ExtensionNameAndPrefix > &extensionNameToPrefix) const override
Definition: ArmnnPreparedModel.cpp:585
armnn::IRuntime::GetInputTensorInfo
TensorInfo GetInputTensorInfo(NetworkId networkId, LayerBindingId layerId) const
Definition: Runtime.cpp:82
armnn::IRuntime::EnqueueWorkload
Status EnqueueWorkload(NetworkId networkId, const InputTensors &inputTensors, const OutputTensors &outputTensors, std::vector< ImportedInputId > preImportedInputIds={}, std::vector< ImportedOutputId > preImportedOutputIds={})
Evaluates a network using input in inputTensors and outputs filled into outputTensors.
Definition: Runtime.cpp:113
armnn::ConstTensor
A tensor defined by a TensorInfo (shape and data type) and an immutable backing store.
Definition: Tensor.hpp:329
armnn_driver::ArmnnPreparedModel::ExecuteGraph
ErrorStatus ExecuteGraph(std::shared_ptr< std::vector< android::nn::RunTimePoolInfo >> &pMemPools, armnn::InputTensors &inputTensors, armnn::OutputTensors &outputTensors, CanonicalExecutionContext callback, const bool pointerMemory=false) const
execute the graph prepared from the request
Definition: ArmnnPreparedModel.cpp:374
armnn_driver::ArmnnPreparedModel::configureExecutionBurst
GeneralResult< SharedBurst > configureExecutionBurst() const override
Definition: ArmnnPreparedModel.cpp:599
armnn::TensorInfo::SetConstant
void SetConstant(const bool IsConstant=true)
Marks the data corresponding to this tensor info as constant.
Definition: Tensor.cpp:518
armnn::IRuntime::ImportOutputs
std::vector< ImportedOutputId > ImportOutputs(NetworkId networkId, const OutputTensors &outputTensors, MemorySource forceImportMemorySource=MemorySource::Undefined)
ImportOutputs separates the importing and mapping of OutputTensors from network execution.
Definition: Runtime.cpp:98
android
Definition: support_library_service.cpp:10