ArmNN
 24.08
LoadedNetwork Class Reference

#include <LoadedNetwork.hpp>

Public Types

using WorkloadQueue = std::vector< std::unique_ptr< IWorkload > >
 

Public Member Functions

 ~LoadedNetwork ()
 
std::unique_ptr< IWorkingMemHandleCreateWorkingMemHandle (NetworkId networkId)
 Create a new unique WorkingMemHandle object. More...
 
TensorInfo GetInputTensorInfo (LayerBindingId layerId) const
 
TensorInfo GetOutputTensorInfo (LayerBindingId layerId) const
 
std::vector< ImportedInputIdImportInputs (const InputTensors &inputTensors, MemorySource forceImportMemorySource=MemorySource::Undefined)
 
std::vector< ImportedOutputIdImportOutputs (const OutputTensors &outputTensors, MemorySource forceImportMemorySource=MemorySource::Undefined)
 
void ClearImportedInputs (const std::vector< ImportedInputId > inputIds)
 
void ClearImportedOutputs (const std::vector< ImportedOutputId > outputIds)
 
Status EnqueueWorkload (const InputTensors &inputTensors, const OutputTensors &outputTensors, std::vector< ImportedInputId > preImportedInputIds={}, std::vector< ImportedOutputId > preImportedOutputIds={})
 Single thread execution of the loaded network. More...
 
Status Execute (const InputTensors &inputTensors, const OutputTensors &outputTensors, IWorkingMemHandle &workingMemHandle, std::vector< ImportedInputId > preImportedInputs={}, std::vector< ImportedOutputId > preImportedOutputs={})
 Thread safe execution of the loaded network. More...
 
const std::shared_ptr< IProfiler > & GetProfiler () const
 
void FreeWorkingMemory ()
 
void RegisterDebugCallback (const DebugCallbackFunction &func)
 
void SendNetworkStructure (arm::pipe::IProfilingService &profilingService)
 
bool IsAsyncEnabled ()
 
arm::pipe::ProfilingGuid GetNetworkGuid ()
 

Static Public Member Functions

static std::unique_ptr< LoadedNetworkMakeLoadedNetwork (std::unique_ptr< IOptimizedNetwork > net, std::string &errorMessage, const INetworkProperties &networkProperties, arm::pipe::IProfilingService *profilingService)
 

Detailed Description

Definition at line 42 of file LoadedNetwork.hpp.

Member Typedef Documentation

◆ WorkloadQueue

using WorkloadQueue = std::vector<std::unique_ptr<IWorkload> >

Definition at line 45 of file LoadedNetwork.hpp.

Constructor & Destructor Documentation

◆ ~LoadedNetwork()

~LoadedNetwork ( )
inline

Definition at line 47 of file LoadedNetwork.hpp.

48  {
50  }

References LoadedNetwork::FreeWorkingMemory().

Member Function Documentation

◆ ClearImportedInputs()

void ClearImportedInputs ( const std::vector< ImportedInputId inputIds)

Definition at line 1761 of file LoadedNetwork.cpp.

1762 {
1763  for (auto id : inputIds)
1764  {
1765  if (id > m_PreImportedInputHandles.size())
1766  {
1767  throw InvalidArgumentException(fmt::format("ClearImportedInputs::Unknown ImportedInputId: {}", id));
1768  }
1769 
1770  auto& importedTensorHandle = m_PreImportedInputHandles[id].m_TensorHandle;
1771  if (!importedTensorHandle)
1772  {
1773  throw InvalidArgumentException(
1774  fmt::format("ClearImportedInputs::ImportedInput with id: {} has already been deleted", id));
1775  }
1776  // Call Unimport then destroy the tensorHandle
1777  importedTensorHandle->Unimport();
1778  importedTensorHandle = {};
1779  }
1780 }

Referenced by RuntimeImpl::ClearImportedInputs().

◆ ClearImportedOutputs()

void ClearImportedOutputs ( const std::vector< ImportedOutputId outputIds)

Definition at line 1782 of file LoadedNetwork.cpp.

1783 {
1784  for (auto id : outputIds)
1785  {
1786  if (id > m_PreImportedOutputHandles.size())
1787  {
1788  throw InvalidArgumentException(fmt::format("ClearImportedOutputs::Unknown ImportedOutputId: {}", id));
1789  }
1790 
1791  auto& importedTensorHandle = m_PreImportedOutputHandles[id].m_TensorHandle;
1792  if (!importedTensorHandle)
1793  {
1794  throw InvalidArgumentException(
1795  fmt::format("ClearImportedOutputs::ImportedOutput with id: {} has already been deleted", id));
1796  }
1797  // Call Unimport then destroy the tensorHandle
1798  importedTensorHandle->Unimport();
1799  importedTensorHandle = {};
1800  }
1801 }

Referenced by RuntimeImpl::ClearImportedOutputs().

◆ CreateWorkingMemHandle()

std::unique_ptr< IWorkingMemHandle > CreateWorkingMemHandle ( NetworkId  networkId)

Create a new unique WorkingMemHandle object.

Create multiple handles if you wish to have overlapped Execution by calling this function from different threads.

Definition at line 2025 of file LoadedNetwork.cpp.

2026 {
2027  Graph& order = m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph();
2028 
2029  // Tensors that will need to be allocated internally within armnn
2030  std::vector<std::unique_ptr<ITensorHandle>> managedTensorHandles;
2031  // Tensors that will be allocated externally by the user
2032  std::vector<std::unique_ptr<ITensorHandle>> unmanagedTensorHandles;
2033 
2034  std::vector<WorkingMemDescriptor> workingMemDescriptors;
2035  std::vector<std::pair<BackendId, ExecutionData>> executionDataVec;
2036 
2037  auto GetTensorHandle = [&](Layer* layer, const OutputSlot& outputSlot)
2038  {
2039  ITensorHandleFactory::FactoryId factoryId = outputSlot.GetTensorHandleFactoryId();
2040  const TensorInfo& tensorInfo = outputSlot.GetTensorInfo();
2041 
2042  if (factoryId == ITensorHandleFactory::LegacyFactoryId)
2043  {
2044  BackendId id = layer->GetBackendId();
2046  return m_WorkloadFactories.at(id)->CreateTensorHandle(tensorInfo, false);
2048  }
2049  else
2050  {
2051  ITensorHandleFactory* handleFactory = m_TensorHandleFactoryRegistry.GetFactory(factoryId);
2052  if (!handleFactory)
2053  {
2054  throw armnn::NullPointerException("handleFactory must not be null.");
2055  }
2056  return handleFactory->CreateTensorHandle(tensorInfo, false);
2057  }
2058  };
2059 
2060  struct HandleInfo
2061  {
2062  ITensorHandle* m_TensorHandle;
2063 
2064  bool m_IsInputLayerHandle = false;
2065  bool m_IsOutputLayerHandle = false;
2066 
2067  WorkingMemHandle::InputMemDescriptorCoords m_InputMemDescriptorCoords;
2068  WorkingMemHandle::OutputMemDescriptorCoords m_OutputMemDescriptorCoords;
2069  };
2070 
2071  std::unordered_map<const OutputSlot*, HandleInfo> outputToHandleInfoMap;
2072 
2073  unsigned int layerIndex = 0;
2074  for (auto&& layer : order)
2075  {
2076  // Constant layers execution and management is handled during loaded network construction
2077  if (layer->GetType() == LayerType::Constant)
2078  {
2079  continue;
2080  }
2081 
2082  WorkingMemDescriptor workingMemDescriptor;
2083 
2084  bool isMemoryManaged = true;
2085  bool isInputLayer = false;
2086  bool isOutputLayer = false;
2087  bool isConnectedToOutputLayer = false;
2088 
2089  if (layer->GetType() == LayerType::Input || layer->GetType() == LayerType::MemImport)
2090  {
2091  // Input layers/workloads will not be executed so the descriptor is not added to workingMemDescriptors
2092  // However we will still need to manage the tensorHandle
2093  isInputLayer = true;
2094  isMemoryManaged = m_NetworkProperties.m_InputSource == MemorySource::Undefined;
2095  }
2096  else if (layer->GetType() == LayerType::Output)
2097  {
2098  isOutputLayer = true;
2099  }
2100 
2101  unsigned int slotIndex = 0;
2102  // Create a tensor handle for each output slot of a layer
2103  // Once we create it, we start managing its lifetime
2104  for (auto& slot : layer->GetOutputSlots())
2105  {
2106  for (unsigned int i = 0; i < slot.GetNumConnections(); ++i)
2107  {
2108  if ((slot.GetConnection(i)->GetOwningLayer().GetType() == LayerType::Output))
2109  {
2110  if (!isConnectedToOutputLayer)
2111  {
2112  isConnectedToOutputLayer = true;
2113  // If Export is enabled disable memory management, so we can export, otherwise we do a copy
2114  isMemoryManaged = m_NetworkProperties.m_OutputSource == MemorySource::Undefined;
2115  }
2116  else
2117  {
2118  // Importing in this case would likely cause unexpected behaviour, so we disallow it.
2119  ARMNN_LOG(warning) <<
2120  fmt::format("Layer name: '{0}' guid: '{1}' has two or more OutputLayers connected to it. "
2121  "This will prevent importing on the connected OutputLayers.",
2122  layer->GetName(), layer->GetGuid());
2123  isMemoryManaged = true;
2124  }
2125  }
2126  }
2127 
2128  ITensorHandle* tensorHandle;
2129  if (isMemoryManaged)
2130  {
2131  managedTensorHandles.emplace_back(GetTensorHandle(layer, slot));
2132  tensorHandle = managedTensorHandles.back().get();
2133  }
2134  else
2135  {
2136  unmanagedTensorHandles.emplace_back(GetTensorHandle(layer, slot));
2137  tensorHandle = unmanagedTensorHandles.back().get();
2138  }
2139 
2140  workingMemDescriptor.m_Outputs.push_back(tensorHandle);
2141 
2142  HandleInfo& handleInfo = outputToHandleInfoMap[&slot];
2143  handleInfo.m_TensorHandle = tensorHandle;
2144 
2145  // Store the coordinates of the current layer's OutputSlot that is connected to the OutputLayer
2146  if (isConnectedToOutputLayer)
2147  {
2148  handleInfo.m_IsOutputLayerHandle = true;
2149  handleInfo.m_OutputMemDescriptorCoords.m_OutputSlotCoords = {layerIndex, slotIndex};
2150  }
2151  // Store the LayerBindingId of the InputLayer
2152  if (isInputLayer)
2153  {
2154  handleInfo.m_IsInputLayerHandle = true;
2155  LayerBindingId bindingId = static_cast<BindableLayer*>(layer)->GetBindingId();
2156  handleInfo.m_InputMemDescriptorCoords.m_LayerBindingId = bindingId;
2157  }
2158  slotIndex++;
2159  }
2160  // Loop through the input slots in the same layer and decrement the reference counter associated
2161  // to each tensor handle we encounter.
2162  // Once it reaches zero, the lifetime of the tensor handle has ended, and we mark its memory as available
2163  // so that the next tensor handle with a non overlapping lifetime can share its memory.
2164  for (auto& slot : layer->GetInputSlots())
2165  {
2166  if (!slot.GetConnection())
2167  {
2168  throw armnn::GraphValidationException("slot must be a valid input slot.");
2169  }
2170 
2171  auto outputSlot = slot.GetConnectedOutputSlot();
2172  auto key = outputSlot->GetOwningLayer().GetGuid();
2173 
2174  // Constant layers execution and management is handled during loaded network construction
2175  auto found = m_ConstantTensorHandles.find(key);
2176  if (found != m_ConstantTensorHandles.end())
2177  {
2178  ITensorHandle* tensorHandle = found->second;
2179  if (slot.IsTensorInfoOverridden())
2180  {
2181  ITensorHandle* decorated = tensorHandle->DecorateTensorHandle(slot.GetTensorInfo()).get();
2182  if (decorated)
2183  {
2184  tensorHandle = decorated;
2185  }
2186  }
2187  workingMemDescriptor.m_Inputs.push_back(tensorHandle);
2188 
2189  // Odd case where a constant layer is connected to an output layer
2190  // We will need to create a HandleInfo to track it
2191  if (isOutputLayer)
2192  {
2193  LayerBindingId bindingId = static_cast<BindableLayer*>(layer)->GetBindingId();
2194 
2195  HandleInfo& handleInfo = outputToHandleInfoMap[outputSlot];
2196  handleInfo.m_TensorHandle = tensorHandle;
2197  handleInfo.m_IsOutputLayerHandle = true;
2198  handleInfo.m_OutputMemDescriptorCoords.m_LayerBindingIds.push_back(bindingId);
2199  handleInfo.m_OutputMemDescriptorCoords.m_InputSlotCoords.push_back({layerIndex, 0});
2200  }
2201  continue;
2202  }
2203 
2204  HandleInfo& handleInfo = outputToHandleInfoMap.at(outputSlot);
2205 
2206  ITensorHandle* inputTensorHandle = handleInfo.m_TensorHandle;
2207  if (slot.IsTensorInfoOverridden())
2208  {
2209  ITensorHandle* decorated = inputTensorHandle->DecorateTensorHandle(slot.GetTensorInfo()).get();
2210  if (decorated)
2211  {
2212  inputTensorHandle = decorated;
2213  }
2214  }
2215  workingMemDescriptor.m_Inputs.push_back(inputTensorHandle);
2216 
2217  // Store the LayerBindingId of the OutputLayer
2218  if (isOutputLayer)
2219  {
2220  LayerBindingId bindingId = static_cast<BindableLayer*>(layer)->GetBindingId();
2221  handleInfo.m_OutputMemDescriptorCoords.m_LayerBindingIds.push_back(bindingId);
2222  handleInfo.m_OutputMemDescriptorCoords.m_InputSlotCoords.push_back({layerIndex, 0});
2223  }
2224  // In this case the layer is not an Output Layer but shares its input tensorhandle with an OutputLayer
2225  // It will need to be updated as well, if we swap out the tensorhandle
2226  else if (handleInfo.m_IsOutputLayerHandle)
2227  {
2228  handleInfo.m_OutputMemDescriptorCoords.m_InputSlotCoords.push_back({layerIndex, slot.GetSlotIndex()});
2229  }
2230 
2231  // Store the coordinates of the InputSlots connected to the InputLayer
2232  // There can be more than one InputSlot connected to an InputLayer, so we use a vector
2233  if (handleInfo.m_IsInputLayerHandle)
2234  {
2235  std::pair<LayerGuid, unsigned int> connectionLocation{layerIndex, slot.GetSlotIndex()};
2236  handleInfo.m_InputMemDescriptorCoords.m_InputSlotCoords.emplace_back(connectionLocation);
2237  }
2238  }
2239 
2240  // Input/Output layers/workloads will not be executed, so the descriptor is not added to workingMemDescriptors
2241  // However we will still need to manage the tensorHandle
2242  if (!isInputLayer)
2243  {
2244  // Simply auto initialise ExecutionData here, so it's added only for the layer that require execution.
2245  // The memory and data will be allocated/assigned for the void* in WorkingMemHandle::Allocate.
2246  std::pair<BackendId, ExecutionData> dataPair;
2247  dataPair.first = layer->GetBackendId();
2248 
2249  executionDataVec.push_back(dataPair);
2250  workingMemDescriptors.push_back(workingMemDescriptor);
2251 
2252  layerIndex++;
2253  }
2254  }
2255 
2256  std::vector<std::pair<std::shared_ptr<TensorMemory>, MemorySource>> tensorMemory;
2257 
2258  auto externalMemoryManager = CreateExternalMemoryManger(tensorMemory);
2259 
2260  // Sort m_TensorMemory, so it's order matches the outputSlot order
2261  std::sort(tensorMemory.begin(), tensorMemory.end(),
2262  [](const std::pair<std::shared_ptr<TensorMemory>, MemorySource>& lhs,
2263  const std::pair<std::shared_ptr<TensorMemory>, MemorySource>& rhs)
2264  {
2265  return lhs.first->m_OutputSlotId < rhs.first->m_OutputSlotId;
2266  });
2267 
2268  std::vector<WorkingMemHandle::InputMemDescriptorCoords> inputConnectionsInfo;
2269  std::vector<WorkingMemHandle::OutputMemDescriptorCoords> outputConnectionsInfo;
2270 
2271  for (const auto& handleInfo: outputToHandleInfoMap)
2272  {
2273  if (handleInfo.second.m_IsOutputLayerHandle)
2274  {
2275  outputConnectionsInfo.emplace_back(handleInfo.second.m_OutputMemDescriptorCoords);
2276  }
2277 
2278  if (handleInfo.second.m_IsInputLayerHandle)
2279  {
2280  inputConnectionsInfo.emplace_back(handleInfo.second.m_InputMemDescriptorCoords);
2281  }
2282  }
2283 
2284  return std::make_unique<WorkingMemHandle>(networkId,
2285  inputConnectionsInfo,
2286  outputConnectionsInfo,
2287  workingMemDescriptors,
2288  std::move(externalMemoryManager),
2289  std::move(tensorMemory),
2290  std::move(managedTensorHandles),
2291  std::move(unmanagedTensorHandles),
2292  executionDataVec,
2293  &m_Backends);
2294 }

References ARMNN_NO_DEPRECATE_WARN_BEGIN, ARMNN_NO_DEPRECATE_WARN_END, ITensorHandleFactory::CreateTensorHandle(), Layer::GetBackendId(), TensorHandleFactoryRegistry::GetFactory(), and ITensorHandleFactory::LegacyFactoryId.

Referenced by RuntimeImpl::CreateWorkingMemHandle().

◆ EnqueueWorkload()

Status EnqueueWorkload ( const InputTensors inputTensors,
const OutputTensors outputTensors,
std::vector< ImportedInputId preImportedInputIds = {},
std::vector< ImportedOutputId preImportedOutputIds = {} 
)

Single thread execution of the loaded network.

Definition at line 872 of file LoadedNetwork.cpp.

876 {
877  const Graph& graph = m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph();
878 
879  // Walk graph to determine the order of execution.
880  if (graph.GetNumLayers() < 2)
881  {
882  ARMNN_LOG(warning) << "IRuntime::EnqueueWorkload()::Less than two nodes in graph";
883  return Status::Failure;
884  }
885 
886  // Data that must be kept alive for the entire execution of the workload.
887  WorkloadData workloadData(inputTensors, outputTensors);
888 
889  // Input tensors can be provided as parameters or pre imported. Either way the number of
890  // tensors should match the number of inputs.
891  if (graph.GetNumInputs() != (inputTensors.size() + preImportedInputIds.size()))
892  {
893  throw InvalidArgumentException("Number of inputs provided does not match network.");
894  }
895 
896  // For each input to the network, call EnqueueInput with the data passed by the user.
897  {
899  m_InputQueue.clear();
900  m_InputQueue.reserve(graph.GetNumInputs());
901 
902  unsigned int inputIndex = 0;
903  unsigned int importedInputIdIndex = 0;
904  std::sort(preImportedInputIds.begin(), preImportedInputIds.end());
905  for (const BindableLayer* inputLayer : graph.GetInputLayers())
906  {
907  if (importedInputIdIndex < preImportedInputIds.size() &&
908  inputIndex == preImportedInputIds[importedInputIdIndex])
909  {
910  // Only replace tensorhandles if they have not already been replaced
911  if (!m_IsInputImported[inputIndex])
912  {
913  auto outputTensorHandle = m_PreImportedInputHandles[inputIndex].m_TensorHandle.get();
914 
915  for (const auto& workloadInfo: m_InputWorkloadSlotPairs[inputLayer->GetBindingId()])
916  {
917  auto workload = m_WorkloadQueue[workloadInfo.m_WorkloadIndex].get();
918  workload->ReplaceInputTensorHandle(outputTensorHandle, workloadInfo.m_SlotIndex);
919  }
920  m_IsInputImported[inputIndex] = true;
921  }
922  importedInputIdIndex++;
923  }
924  else
925  {
926  if (m_IsInputImported[inputIndex])
927  {
928  OutputHandler& handler = const_cast<OutputHandler&>(inputLayer->GetOutputHandler(0));
929 
930  for (const auto& workloadInfo: m_InputWorkloadSlotPairs[inputLayer->GetBindingId()])
931  {
932  auto workload = m_WorkloadQueue[workloadInfo.m_WorkloadIndex].get();
933  workload->ReplaceInputTensorHandle(handler.GetData(), workloadInfo.m_SlotIndex);
934  }
935 
936  m_IsInputImported[inputIndex] = false;
937  }
938 
939  // InputTensorHandle is not imported yet, process to enqueue input
940  const TensorPin& pin = workloadData.GetInputTensorPin(inputLayer->GetBindingId());
941  EnqueueInput(*inputLayer, pin.GetTensorHandle(), pin.GetTensorInfo());
942  }
943  inputIndex++;
944  }
945  }
946  // For each output to the network, call EnqueueOutput with the data passed by the user.
947  {
949  m_OutputQueue.clear();
950  m_OutputQueue.reserve(graph.GetNumOutputs());
951 
952  if (preImportedOutputIds.size() > graph.GetNumOutputs())
953  {
954  throw InvalidArgumentException("Invalid number of preImportedOutputIds");
955  }
956 
957  unsigned int outputIndex = 0;
958  unsigned int importedOutputIdIndex = 0;
959  std::sort(preImportedOutputIds.begin(), preImportedOutputIds.end());
960  for (const BindableLayer* outputLayer : graph.GetOutputLayers())
961  {
962  if (importedOutputIdIndex < preImportedOutputIds.size() &&
963  outputIndex == preImportedOutputIds[importedOutputIdIndex])
964  {
965  // Only replace tensorhandles if they have not already been replaced
966  ITensorHandle* inputTensorHandle = m_PreImportedOutputHandles[outputIndex].m_TensorHandle.get();
967 
968  if (!m_IsOutputImported[outputIndex])
969  {
970  const auto bindingId = outputLayer->GetBindingId();
971  const auto& indices = m_OutputWorkloadSlotPairs[bindingId];
972 
973  auto outputWorkload = m_WorkloadQueue[indices.m_OutputSlotIndices.m_WorkloadIndex].get();
974 
975  outputWorkload->ReplaceOutputTensorHandle(inputTensorHandle,
976  indices.m_OutputSlotIndices.m_SlotIndex);
977 
978  for (const auto& workloadInfo: indices.m_InputSlotIndices)
979  {
980  auto inputWorkload = m_WorkloadQueue[workloadInfo.m_WorkloadIndex].get();
981  inputWorkload->ReplaceInputTensorHandle(inputTensorHandle, workloadInfo.m_SlotIndex);
982  }
983  m_IsOutputImported[outputIndex] = true;
984  }
985 
986  if (!inputTensorHandle)
987  {
988  throw armnn::NullPointerException("Data should have been allocated.");
989  }
990 
991  MemSyncQueueDescriptor syncDesc;
992  syncDesc.m_Inputs.push_back(inputTensorHandle);
993  WorkloadInfo info;
994  info.m_InputTensorInfos.push_back(outputLayer->GetInputSlot(0).GetTensorInfo());
995 
996  auto syncWorkload = std::make_unique<SyncMemGenericWorkload>(syncDesc, info);
997  if (!syncWorkload)
998  {
999  throw armnn::NullPointerException("No sync workload created");
1000  }
1001 
1002  m_OutputQueue.push_back(std::move(syncWorkload));
1003  importedOutputIdIndex++;
1004  }
1005  else
1006  {
1007  if (m_IsOutputImported[outputIndex])
1008  {
1009  const auto bindingId = outputLayer->GetBindingId();
1010  const auto& indices = m_OutputWorkloadSlotPairs[bindingId];
1011 
1012  auto outputWorkload = m_WorkloadQueue[indices.m_OutputSlotIndices.m_WorkloadIndex].get();
1013  const OutputHandler& outputHandler =
1014  outputLayer->GetInputSlot(0).GetConnectedOutputSlot()->GetOutputHandler();
1015 
1016  outputWorkload->ReplaceOutputTensorHandle(
1017  outputHandler.GetData(), indices.m_OutputSlotIndices.m_SlotIndex);
1018 
1019  for (const auto& workloadInfo: indices.m_InputSlotIndices)
1020  {
1021  auto inputWorkload = m_WorkloadQueue[workloadInfo.m_WorkloadIndex].get();
1022  inputWorkload->ReplaceInputTensorHandle(outputHandler.GetData(), workloadInfo.m_SlotIndex);
1023  }
1024  m_IsOutputImported[outputIndex] = false;
1025  }
1026 
1027  const TensorPin& pin = workloadData.GetOutputTensorPin(outputLayer->GetBindingId());
1028  // OutputTensorHandle is not imported yet, process to enqueue Output
1029  EnqueueOutput(*outputLayer, pin.GetTensorHandle(), pin.GetTensorInfo());
1030  }
1031  outputIndex++;
1032  }
1033  }
1034 
1035  std::unique_ptr<TimelineUtilityMethods> timelineUtils =
1036  TimelineUtilityMethods::GetTimelineUtils(*m_ProfilingService);
1037  ProfilingGuid inferenceGuid = m_ProfilingService->GetNextGuid();
1038  if (timelineUtils)
1039  {
1040  // Add inference timeline trace if profiling is enabled.
1041  ProfilingGuid networkGuid = m_OptimizedNetwork->GetGuid();
1042  timelineUtils->CreateTypedEntity(inferenceGuid, LabelsAndEventClasses::INFERENCE_GUID);
1043  timelineUtils->CreateRelationship(ProfilingRelationshipType::RetentionLink,
1044  networkGuid,
1045  inferenceGuid,
1046  LabelsAndEventClasses::EXECUTION_OF_GUID);
1047  timelineUtils->RecordEvent(inferenceGuid, LabelsAndEventClasses::ARMNN_PROFILING_SOL_EVENT_CLASS);
1048  }
1049 
1050  bool executionSucceeded = true;
1051 
1052  {
1053  if (m_ProfilingService->IsProfilingEnabled())
1054  {
1055  m_ProfilingService->IncrementCounterValue(INFERENCES_RUN);
1056  }
1058  ARMNN_SCOPED_HEAP_PROFILING("Executing");
1059  executionSucceeded = Execute(timelineUtils, inferenceGuid);
1060  }
1061 
1062  if (timelineUtils)
1063  {
1064  // Add end of life of the inference timeline if profiling is enabled.
1065  timelineUtils->RecordEvent(inferenceGuid, LabelsAndEventClasses::ARMNN_PROFILING_EOL_EVENT_CLASS);
1066  timelineUtils->Commit();
1067  }
1068 
1069  return executionSucceeded ? Status::Success : Status::Failure;
1070 }

References ARMNN_LOG, ARMNN_SCOPED_PROFILING_EVENT, armnn::Failure, OutputHandler::GetData(), Graph::GetInputLayers(), Graph::GetNumInputs(), Graph::GetNumLayers(), Graph::GetNumOutputs(), Graph::GetOutputLayers(), armnn::info, QueueDescriptor::m_Inputs, armnn::Undefined, and armnn::warning.

Referenced by RuntimeImpl::EnqueueWorkload().

◆ Execute()

Status Execute ( const InputTensors inputTensors,
const OutputTensors outputTensors,
IWorkingMemHandle workingMemHandle,
std::vector< ImportedInputId preImportedInputs = {},
std::vector< ImportedOutputId preImportedOutputs = {} 
)

Thread safe execution of the loaded network.

Definition at line 1803 of file LoadedNetwork.cpp.

1808 {
1809  const Graph& graph = m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph();
1810 
1811  if (inputTensors.size() + preImportedInputs.size() != graph.GetNumInputs())
1812  {
1813  if (preImportedInputs.empty())
1814  {
1815  throw InvalidArgumentException("LoadedNetwork::Execute: Number of inputs provided does not match network.");
1816  }
1817  else
1818  {
1819  throw InvalidArgumentException("LoadedNetwork::Execute: "
1820  "Number of inputs + preImportedInputs provided does not match network.");
1821  }
1822  }
1823 
1824  if (outputTensors.size() + preImportedOutputs.size() != graph.GetNumOutputs())
1825  {
1826  if (preImportedOutputs.empty())
1827  {
1828  throw InvalidArgumentException("LoadedNetwork::Execute: "
1829  "Number of outputs provided does not match network.");
1830  }
1831  else
1832  {
1833  throw InvalidArgumentException("LoadedNetwork::Execute: "
1834  "Number of outputs + preImportedOutputs provided does not match network.");
1835  }
1836  }
1837 
1838  WorkingMemHandle& workingMemHandle = dynamic_cast<WorkingMemHandle&>(iWorkingMemHandle);
1839  // Collect all the given LayerBindingIds and check them for duplicates and unknowns.
1840  std::vector<LayerBindingId>& bindingIds = workingMemHandle.GetBindingIdVector();
1841  unsigned int index = 0;
1842  for (auto pair : inputTensors)
1843  {
1844  bindingIds[index++] = pair.first;
1845  }
1846  for (ImportedInputId id : preImportedInputs)
1847  {
1848  bindingIds[index++] = ValidateImportedInputID(id);
1849  }
1850  for (auto pair : outputTensors)
1851  {
1852  bindingIds[index++] = pair.first;
1853  }
1854  for (ImportedOutputId id : preImportedOutputs)
1855  {
1856  bindingIds[index++] = ValidateImportedOutputID(id);
1857  }
1858 
1859  workingMemHandle.ValidateBindingIds();
1860 
1861  auto resetMemHandle = [&]()
1862  {
1863  for (ImportedInputId id: preImportedInputs)
1864  {
1865  const LayerBindingId layerBindingId = m_PreImportedInputHandles[id].m_LayerBindingId;
1866 
1867  auto inputHandle = workingMemHandle.GetInputHandle(layerBindingId);
1868  auto inputConnections = workingMemHandle.GetInputConnections(layerBindingId);
1869  for (auto it : inputConnections)
1870  {
1871  *it = inputHandle;
1872  }
1873  }
1874 
1875  for (ImportedOutputId id: preImportedOutputs)
1876  {
1877  const LayerBindingId layerBindingId = m_PreImportedOutputHandles[id].m_LayerBindingId;
1878 
1879  auto outputHandle = workingMemHandle.GetOutputHandle(layerBindingId);
1880  auto outputConnections = workingMemHandle.GetOutputConnection(layerBindingId);
1881 
1882  for (auto it : outputConnections)
1883  {
1884  *it = outputHandle;
1885  }
1886  }
1887  };
1888 
1889  std::unique_ptr<TimelineUtilityMethods> timelineUtils =
1890  TimelineUtilityMethods::GetTimelineUtils(*m_ProfilingService);
1891  ProfilingGuid inferenceGuid = m_ProfilingService->GetNextGuid();
1892  if (timelineUtils)
1893  {
1894  // Add inference timeline trace if profiling is enabled.
1895  ProfilingGuid networkGuid = m_OptimizedNetwork->GetGuid();
1896  timelineUtils->CreateTypedEntity(inferenceGuid,LabelsAndEventClasses::INFERENCE_GUID);
1897  timelineUtils->CreateRelationship(ProfilingRelationshipType::RetentionLink,
1898  networkGuid,
1899  inferenceGuid,
1900  LabelsAndEventClasses::EXECUTION_OF_GUID);
1901  timelineUtils->RecordEvent(inferenceGuid,LabelsAndEventClasses::ARMNN_PROFILING_SOL_EVENT_CLASS);
1902  }
1903 
1904  bool executionSucceeded = true;
1905 
1906  if (timelineUtils)
1907  {
1908  // Add end of life of the inference timeline if profiling is enabled.
1909  timelineUtils->RecordEvent(inferenceGuid,LabelsAndEventClasses::ARMNN_PROFILING_EOL_EVENT_CLASS);
1910  timelineUtils->Commit();
1911  }
1912 
1913  if (!workingMemHandle.IsAllocated())
1914  {
1915  workingMemHandle.Allocate();
1916  }
1917 
1918  {
1920  for (auto pair : inputTensors)
1921  {
1922  EnqueueInput(pair.second, workingMemHandle.GetInputHandle(pair.first));
1923  }
1924 
1925  // Swap in the pre-imported inputs if any
1926  for (ImportedInputId id : preImportedInputs)
1927  {
1928  const ImportedTensorHandlePin& importedInputPin = m_PreImportedInputHandles[id];
1929  const LayerBindingId layerBindingId = m_PreImportedInputHandles[id].m_LayerBindingId;
1930  const auto& preimportedHandle = importedInputPin.m_TensorHandle;
1931 
1932  auto inputConnections = workingMemHandle.GetInputConnections(layerBindingId);
1933  for (auto it : inputConnections)
1934  {
1935  *it = preimportedHandle.get();
1936  }
1937  }
1938  }
1939  {
1941  if (m_NetworkProperties.m_OutputSource != MemorySource::Undefined)
1942  {
1943  for (auto pair: outputTensors)
1944  {
1945  ImportOutputTensor(pair.second, workingMemHandle.GetOutputHandle(pair.first));
1946  }
1947  }
1948 
1949  for (ImportedOutputId id : preImportedOutputs)
1950  {
1951  const ImportedTensorHandlePin& importedOutputPin = m_PreImportedOutputHandles[id];
1952  const LayerBindingId layerBindingId = m_PreImportedOutputHandles[id].m_LayerBindingId;
1953  const auto& preimportedHandle = importedOutputPin.m_TensorHandle;
1954 
1955  auto outputConnections = workingMemHandle.GetOutputConnection(layerBindingId);
1956  for (auto it : outputConnections)
1957  {
1958  *it = preimportedHandle.get();
1959  }
1960  }
1961  }
1962 
1963  auto Fail = [&](const std::exception& error)
1964  {
1965  ARMNN_LOG(error) << "An error occurred attempting to execute a workload: " << error.what();
1966  executionSucceeded = false;
1967  };
1968  ProfilingDynamicGuid workloadInferenceID(0);
1969 
1970  try
1971  {
1972  for (unsigned int i = 0; i < m_WorkloadQueue.size(); ++i)
1973  {
1974  auto& workload = m_WorkloadQueue[i];
1975  if (timelineUtils)
1976  {
1977  workloadInferenceID = timelineUtils->RecordWorkloadInferenceAndStartOfLifeEvent(workload->GetGuid(),
1978  inferenceGuid);
1979  }
1980 
1981  workload->ExecuteAsync(workingMemHandle.GetExecutionDataAt(i).second);
1982 
1983  if (timelineUtils)
1984  {
1985  timelineUtils->RecordEndOfLifeEvent(workloadInferenceID);
1986  }
1987  }
1988  }
1989  catch (const RuntimeException& error)
1990  {
1991  resetMemHandle();
1992  Fail(error);
1993  }
1994  catch (const std::runtime_error& error)
1995  {
1996  resetMemHandle();
1997  Fail(error);
1998  }
1999  catch (...)
2000  {
2001  resetMemHandle();
2002  throw;
2003  }
2004 
2005  if (m_NetworkProperties.m_OutputSource == MemorySource::Undefined)
2006  {
2007  for (auto pair: outputTensors)
2008  {
2009  CopyToOutputTensor(pair.second, workingMemHandle.GetOutputHandle(pair.first));
2010  }
2011  }
2012  else
2013  {
2014  ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "SyncMemGeneric_Execute");
2015  workingMemHandle.MemSyncOutputs();
2016  }
2017 
2018  resetMemHandle();
2019 
2020  return executionSucceeded ? Status::Success : Status::Failure;
2021 }

References WorkingMemHandle::Allocate(), ARMNN_LOG, ARMNN_SCOPED_PROFILING_EVENT, armnn::CopyToOutputTensor(), armnn::error, armnn::Failure, WorkingMemHandle::GetBindingIdVector(), WorkingMemHandle::GetExecutionDataAt(), WorkingMemHandle::GetInputConnections(), WorkingMemHandle::GetInputHandle(), Graph::GetNumInputs(), Graph::GetNumOutputs(), WorkingMemHandle::GetOutputConnection(), WorkingMemHandle::GetOutputHandle(), WorkingMemHandle::IsAllocated(), INetworkProperties::m_OutputSource, WorkingMemHandle::MemSyncOutputs(), armnn::Success, armnn::Undefined, and WorkingMemHandle::ValidateBindingIds().

Referenced by RuntimeImpl::Execute().

◆ FreeWorkingMemory()

void FreeWorkingMemory ( )

Definition at line 1286 of file LoadedNetwork.cpp.

1287 {
1288 #if !defined(ARMNN_DISABLE_THREADS)
1289  std::lock_guard<std::mutex> lockGuard(m_WorkingMemMutex);
1290 #endif
1291 
1292  if (!m_IsWorkingMemAllocated)
1293  {
1294  return;
1295  }
1296 
1297  if (m_ExternalMemoryManager)
1298  {
1299  m_ExternalMemoryManager->Deallocate();
1300  }
1301 
1302  // Informs the memory managers to release memory in its respective memory group
1303  for (auto&& memoryManager : m_BackendMemoryMangers)
1304  {
1305  if (memoryManager)
1306  {
1307  memoryManager->Release();
1308  }
1309  }
1310  m_TensorHandleFactoryRegistry.ReleaseMemory();
1311  m_IsWorkingMemAllocated = false;
1312 }

References TensorHandleFactoryRegistry::ReleaseMemory().

Referenced by RuntimeImpl::CreateWorkingMemHandle(), RuntimeImpl::EnqueueWorkload(), and LoadedNetwork::~LoadedNetwork().

◆ GetInputTensorInfo()

TensorInfo GetInputTensorInfo ( LayerBindingId  layerId) const

Definition at line 715 of file LoadedNetwork.cpp.

716 {
717  for (auto&& inputLayer : m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph().GetInputLayers())
718  {
719  if (inputLayer->GetNumOutputSlots() != 1)
720  {
721  throw armnn::GraphValidationException("Input layer should have exactly 1 output slot");
722  }
723 
724  if (inputLayer->GetBindingId() == layerId)
725  {
726  return inputLayer->GetOutputSlot(0).GetTensorInfo();
727  }
728  }
729 
730  throw InvalidArgumentException(fmt::format("No input layer is associated with id {}", layerId));
731 }

Referenced by RuntimeImpl::GetInputTensorInfo().

◆ GetNetworkGuid()

ProfilingGuid GetNetworkGuid ( )

Definition at line 710 of file LoadedNetwork.cpp.

711 {
712  return m_OptimizedNetwork->GetGuid();
713 }

◆ GetOutputTensorInfo()

TensorInfo GetOutputTensorInfo ( LayerBindingId  layerId) const

Definition at line 733 of file LoadedNetwork.cpp.

734 {
735  for (auto&& outputLayer : m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph().GetOutputLayers())
736  {
737  if (outputLayer->GetNumInputSlots() != 1)
738  {
739  throw armnn::GraphValidationException("Output layer should have exactly 1 input slot");
740  }
741 
742  if (!outputLayer->GetInputSlot(0).GetConnection())
743  {
744  throw armnn::GraphValidationException("Input slot on Output layer must be connected");
745  }
746 
747  if (outputLayer->GetBindingId() == layerId)
748  {
749  return outputLayer->GetInputSlot(0).GetTensorInfo();
750  }
751  }
752 
753  throw InvalidArgumentException(fmt::format("No output layer is associated with id {}", layerId));
754 }

Referenced by RuntimeImpl::GetOutputTensorInfo().

◆ GetProfiler()

const std::shared_ptr<IProfiler>& GetProfiler ( ) const
inline

Definition at line 87 of file LoadedNetwork.hpp.

87 { return m_OptimizedNetwork->GetProfiler(); }

Referenced by RuntimeImpl::CreateWorkingMemHandle(), RuntimeImpl::EnqueueWorkload(), and RuntimeImpl::Execute().

◆ ImportInputs()

std::vector< ImportedInputId > ImportInputs ( const InputTensors inputTensors,
MemorySource  forceImportMemorySource = MemorySource::Undefined 
)

Definition at line 1486 of file LoadedNetwork.cpp.

1488 {
1489  if (!m_NetworkProperties.m_AsyncEnabled)
1490  {
1491  // Cannot import if import is not enabled and forceImportMemorySource is undefined
1492  if (forceImportMemorySource == MemorySource::Undefined)
1493  {
1494  throw MemoryImportException("ImportInputs: Memory Import failed, NetworkProperties.m_ImportEnabled");
1495  }
1496  // The number of pre imported tensors should not exceed the number of inputs.
1497  if (inputTensors.size() > m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph().GetNumInputs())
1498  {
1499  throw MemoryImportException("ImportInputs: The number of tensors provided exceeds the number of inputs.");
1500  }
1501 
1502  std::vector<ImportedInputId> importedInputs;
1503  Graph& graph = m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph().TopologicalSort();
1504  unsigned int inputIndex = 0;
1505  for (const BindableLayer* inputLayer : graph.GetInputLayers())
1506  {
1507  auto outputTensorHandle = m_PreImportedInputHandles[inputIndex].m_TensorHandle.get();
1508 
1509  if (!outputTensorHandle)
1510  {
1511  inputIndex++;
1512  continue;
1513  }
1514 
1515  auto layerBindingId = inputLayer->GetBindingId();
1516  auto it = std::find_if(inputTensors.begin(), inputTensors.end(), [=](const auto& inputTensor)
1517  {
1518  return inputTensor.first == layerBindingId;
1519  });
1520 
1521  if (it == inputTensors.end())
1522  {
1523  inputIndex++;
1524  continue;
1525  }
1526 
1527  const auto& inputTensor = *it;
1528  std::unique_ptr<ITensorHandle> passThroughTensorHandle =
1529  std::make_unique<ConstPassthroughTensorHandle>(inputTensor.second.GetInfo(),
1530  inputTensor.second.GetMemoryArea());
1531 
1532  try
1533  {
1534  if (outputTensorHandle->CanBeImported(passThroughTensorHandle->Map(), forceImportMemorySource)
1535  && (outputTensorHandle->Import(passThroughTensorHandle->Map(), forceImportMemorySource)))
1536  {
1537  importedInputs.push_back(inputIndex);
1538  }
1539  passThroughTensorHandle->Unmap();
1540  }
1541  catch(const MemoryImportException& exception)
1542  {
1543  ARMNN_LOG(error) << "An error occurred attempting to import input_"
1544  << inputIndex << " : " << exception.what();
1545  passThroughTensorHandle->Unmap();
1546  }
1547  inputIndex++;
1548  }
1549 
1550  return importedInputs;
1551  }
1552  else
1553  {
1554  // Import when the import of network properties is enabled
1555  std::vector<ImportedInputId> importedInputs;
1556  Graph& graph = m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph().TopologicalSort();
1557 
1558  for (auto inputTensor : inputTensors)
1559  {
1560  auto layerBindingId = inputTensor.first;
1561  auto it = std::find_if(graph.GetInputLayers().begin(), graph.GetInputLayers().end(), [=](auto* layer)
1562  {
1563  return layer->GetBindingId() == layerBindingId;
1564  });
1565 
1566  if (it == graph.GetInputLayers().end())
1567  {
1568  throw MemoryImportException(fmt::format(
1569  "ImportInputs: Memory Import failed, unknown LayerBindingId: {}", layerBindingId));
1570  }
1571 
1572  const Layer* layer = *it;
1573  if (layer->GetType() != LayerType::Input)
1574  {
1575  throw InvalidArgumentException("ImportInputs: given layer not an InputLayer");
1576  }
1577 
1578  auto& backend = m_Backends.at(layer->GetBackendId());
1579  if (!HasMatchingCapability(BackendOptions::BackendOption{"PreImportIOTensors", true},
1580  backend->GetCapabilities()))
1581  {
1582  std::string er = backend->GetId();
1583  er += " does not have PreImportIOTensors capability";
1584  throw BackendCapabilityException(er);
1585  }
1586 
1587  const OutputSlot& outputSlot = layer->GetOutputSlots()[0];
1588 
1589  ITensorHandleFactory::FactoryId factoryId = outputSlot.GetTensorHandleFactoryId();
1590  const TensorInfo& tensorInfo = outputSlot.GetTensorInfo();
1591 
1592  ITensorHandleFactory* handleFactory = m_TensorHandleFactoryRegistry.GetFactory(factoryId);
1593  if (!handleFactory)
1594  {
1595  throw armnn::NullPointerException("handleFactory must not be null.");
1596  }
1597 
1598  ImportedTensorHandlePin importedTensorHandlePin{layerBindingId,
1599  handleFactory->CreateTensorHandle(tensorInfo, false)};
1600 
1601  ITensorHandle* tensorHandle = importedTensorHandlePin.m_TensorHandle.get();
1602 
1603  if (!CheckFlag(tensorHandle->GetImportFlags(), forceImportMemorySource))
1604  {
1605  throw MemoryImportException(
1606  fmt::format("ImportInputs: Memory Import failed, backend: "
1607  "{} does not support importing from source {}"
1608  , factoryId, m_NetworkProperties.m_InputSource));
1609  }
1610 
1611  std::unique_ptr<ITensorHandle> passThroughTensorHandle =
1612  std::make_unique<ConstPassthroughTensorHandle>(inputTensor.second.GetInfo(),
1613  inputTensor.second.GetMemoryArea());
1614 
1615  if (tensorHandle->Import(passThroughTensorHandle->Map(), forceImportMemorySource))
1616  {
1617  importedInputs.push_back(m_CurImportedInputId++);
1618  passThroughTensorHandle->Unmap();
1619  }
1620  else
1621  {
1622  passThroughTensorHandle->Unmap();
1623  throw MemoryImportException("ImportInputs: Memory Import failed");
1624  }
1625 
1626  m_PreImportedInputHandles.push_back(std::move(importedTensorHandlePin));
1627  }
1628  return importedInputs;
1629  }
1630 }

References ARMNN_LOG, Graph::InputLayersAccessor::begin(), armnn::CheckFlag(), ITensorHandleFactory::CreateTensorHandle(), Graph::InputLayersAccessor::end(), armnn::error, Layer::GetBackendId(), TensorHandleFactoryRegistry::GetFactory(), ITensorHandle::GetImportFlags(), Graph::GetInputLayers(), Layer::GetOutputSlots(), OutputSlot::GetTensorHandleFactoryId(), OutputSlot::GetTensorInfo(), Layer::GetType(), armnn::HasMatchingCapability(), ITensorHandle::Import(), armnn::Input, INetworkProperties::m_AsyncEnabled, INetworkProperties::m_InputSource, armnn::Undefined, and Exception::what().

Referenced by RuntimeImpl::ImportInputs().

◆ ImportOutputs()

std::vector< ImportedOutputId > ImportOutputs ( const OutputTensors outputTensors,
MemorySource  forceImportMemorySource = MemorySource::Undefined 
)

Definition at line 1632 of file LoadedNetwork.cpp.

1634 {
1635  if (!m_NetworkProperties.m_AsyncEnabled)
1636  {
1637  // Cannot import if import is not enabled and forceImportMemorySource is undefined
1638  if (forceImportMemorySource == MemorySource::Undefined)
1639  {
1640  throw MemoryImportException("ImportOutputs: Memory Import failed, NetworkProperties.m_ImportEnabled");
1641  }
1642  // If forceImportMemorySource is defined, try import if memory is aligned
1643  if (outputTensors.size() != m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph().GetNumOutputs())
1644  {
1645  throw MemoryImportException("ImportOutputs: Force Import failed, incorrect number of tensors");
1646  }
1647  std::vector<ImportedOutputId> importedOutputs;
1648  Graph& graph = m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph().TopologicalSort();
1649 
1650  unsigned int outputIndex = 0;
1651  for (const BindableLayer* const outputLayer : graph.GetOutputLayers())
1652  {
1653  auto inputTensorHandle = m_PreImportedOutputHandles[outputIndex].m_TensorHandle.get();
1654  if (!inputTensorHandle)
1655  {
1656  outputIndex++;
1657  continue;
1658  }
1659 
1660  auto layerBindingId = outputLayer->GetBindingId();
1661  auto it = std::find_if(outputTensors.begin(), outputTensors.end(), [=] (const auto& outputTensor)
1662  {
1663  return outputTensor.first == layerBindingId;
1664  });
1665 
1666  if (it == outputTensors.end())
1667  {
1668  outputIndex++;
1669  continue;
1670  }
1671 
1672  const auto outputTensor = *it;
1673  try
1674  {
1675  // Check if the output memory can be imported
1676  if (inputTensorHandle->CanBeImported(outputTensor.second.GetMemoryArea(), forceImportMemorySource)
1677  && inputTensorHandle->Import(outputTensor.second.GetMemoryArea(), forceImportMemorySource))
1678  {
1679  importedOutputs.push_back(outputIndex);
1680  }
1681  }
1682  catch(const MemoryImportException& exception)
1683  {
1684  ARMNN_LOG(error) << "An error occurred attempting to import output_"
1685  << outputIndex << " : " << exception.what();
1686  }
1687  outputIndex++;
1688  }
1689  return importedOutputs;
1690  }
1691 
1692  std::vector<ImportedOutputId> importedOutputs;
1693  Graph& graph = m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph().TopologicalSort();
1694 
1695  for (const auto& outputTensor : outputTensors)
1696  {
1697  auto layerBindingId = outputTensor.first;
1698  auto it = std::find_if(graph.GetOutputLayers().begin(), graph.GetOutputLayers().end(), [=](auto* layer)
1699  {
1700  return layer->GetBindingId() == layerBindingId;
1701  });
1702 
1703  if (it == graph.GetOutputLayers().end())
1704  {
1705  throw MemoryImportException(fmt::format("ImportOutputs: Memory Import failed, unknown LayerBindingId: {}",
1706  layerBindingId));
1707  }
1708 
1709  const Layer* layer = *it;
1710  if (layer->GetType() != LayerType::Output)
1711  {
1712  throw InvalidArgumentException("ImportOutputs: given layer not an OutputLayer");
1713  }
1714 
1715  auto& backend = m_Backends.at(layer->GetBackendId());
1716  if (!HasMatchingCapability(BackendOptions::BackendOption{"PreImportIOTensors", true},
1717  backend->GetCapabilities()))
1718  {
1719  std::string er = backend->GetId();
1720  er += " does not have PreImportIOTensors capability";
1721  throw BackendCapabilityException(er);
1722  }
1723 
1724  const InputSlot& inputSlot = layer->GetInputSlots()[0];
1725  ITensorHandleFactory::FactoryId factoryId = inputSlot.GetConnectedOutputSlot()->GetTensorHandleFactoryId();
1726  const TensorInfo& tensorInfo = inputSlot.GetTensorInfo();
1727 
1728  ITensorHandleFactory* handleFactory = m_TensorHandleFactoryRegistry.GetFactory(factoryId);
1729  if (!handleFactory)
1730  {
1731  throw armnn::NullPointerException("handleFactory must not be null.");
1732  }
1733 
1734  ImportedTensorHandlePin importedTensorHandlePin{layerBindingId,
1735  handleFactory->CreateTensorHandle(tensorInfo, false)};
1736 
1737  ITensorHandle* tensorHandle = importedTensorHandlePin.m_TensorHandle.get();
1738 
1739  if (!CheckFlag(tensorHandle->GetImportFlags(), forceImportMemorySource))
1740  {
1741  throw MemoryImportException(fmt::format("ImportInputs: Memory Import failed, backend: "
1742  "{} does not support importing from source {}"
1743  , factoryId, forceImportMemorySource));
1744  }
1745 
1746  if (tensorHandle->Import(outputTensor.second.GetMemoryArea(), forceImportMemorySource))
1747  {
1748  importedOutputs.push_back(m_CurImportedOutputId++);
1749  }
1750  else
1751  {
1752  throw MemoryImportException("ImportInputs: Memory Import failed");
1753  }
1754 
1755  m_PreImportedOutputHandles.push_back(std::move(importedTensorHandlePin));
1756  }
1757 
1758  return importedOutputs;
1759 }

References ARMNN_LOG, Graph::OutputLayersAccessor::begin(), armnn::CheckFlag(), ITensorHandleFactory::CreateTensorHandle(), Graph::OutputLayersAccessor::end(), armnn::error, Layer::GetBackendId(), InputSlot::GetConnectedOutputSlot(), TensorHandleFactoryRegistry::GetFactory(), ITensorHandle::GetImportFlags(), Layer::GetInputSlots(), Graph::GetOutputLayers(), OutputSlot::GetTensorHandleFactoryId(), InputSlot::GetTensorInfo(), Layer::GetType(), armnn::HasMatchingCapability(), ITensorHandle::Import(), INetworkProperties::m_AsyncEnabled, armnn::Output, armnn::Undefined, and Exception::what().

Referenced by RuntimeImpl::ImportOutputs().

◆ IsAsyncEnabled()

bool IsAsyncEnabled ( )
inline

Definition at line 95 of file LoadedNetwork.hpp.

96  {
97  return m_NetworkProperties.m_AsyncEnabled;
98  }

References INetworkProperties::m_AsyncEnabled.

Referenced by RuntimeImpl::CreateWorkingMemHandle(), RuntimeImpl::EnqueueWorkload(), and RuntimeImpl::Execute().

◆ MakeLoadedNetwork()

std::unique_ptr< LoadedNetwork > MakeLoadedNetwork ( std::unique_ptr< IOptimizedNetwork net,
std::string &  errorMessage,
const INetworkProperties networkProperties,
arm::pipe::IProfilingService *  profilingService 
)
static

Definition at line 173 of file LoadedNetwork.cpp.

177 {
178  std::unique_ptr<LoadedNetwork> loadedNetwork;
179 
180  auto Fail = [&](const std::exception& error) -> std::unique_ptr<LoadedNetwork>
181  {
182  errorMessage = ToErrorMessage("An error occurred when preparing the network workloads: ", error);
183  ARMNN_LOG(error) << errorMessage;
184 
185  return std::unique_ptr<LoadedNetwork>();
186  };
187 
188  try
189  {
190  loadedNetwork.reset(new LoadedNetwork(std::move(net), networkProperties, profilingService));
191  }
192  catch (const armnn::RuntimeException& error)
193  {
194  return Fail(error);
195  }
196  catch (const armnn::Exception& error)
197  {
198  return Fail(error);
199  }
200  catch (const std::runtime_error& error)
201  {
202  return Fail(error);
203  }
204 
205  return loadedNetwork;
206 }

References ARMNN_LOG, and armnn::error.

Referenced by RuntimeImpl::LoadNetwork().

◆ RegisterDebugCallback()

void RegisterDebugCallback ( const DebugCallbackFunction func)

Definition at line 2296 of file LoadedNetwork.cpp.

2297 {
2298  for (auto&& workloadPtr: m_WorkloadQueue)
2299  {
2300  workloadPtr.get()->RegisterDebugCallback(func);
2301  }
2302 }

Referenced by RuntimeImpl::RegisterDebugCallback().

◆ SendNetworkStructure()

void SendNetworkStructure ( arm::pipe::IProfilingService &  profilingService)

Definition at line 672 of file LoadedNetwork.cpp.

673 {
674  ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "LoadNetwork_SendNetworkStructure");
675  Graph& order = m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph().TopologicalSort();
676  ProfilingGuid networkGuid = m_OptimizedNetwork->GetGuid();
677 
678  std::unique_ptr<TimelineUtilityMethods> timelineUtils =
679  TimelineUtilityMethods::GetTimelineUtils(profilingService);
680 
681  timelineUtils->CreateTypedEntity(networkGuid, LabelsAndEventClasses::NETWORK_GUID);
682 
683  for (auto&& layer : order)
684  {
685  // Add layer to the post-optimisation network structure
686  AddLayerStructure(timelineUtils, *layer, networkGuid);
687  switch (layer->GetType())
688  {
689  case LayerType::Input:
690  case LayerType::Output:
691  {
692  // Inputs and outputs are treated in a special way - see EnqueueInput() and EnqueueOutput().
693  break;
694  }
695  default:
696  {
697  for (auto& workload : m_WorkloadQueue)
698  {
699  // Add workload to the post-optimisation network structure
700  AddWorkloadStructure(timelineUtils, workload, *layer);
701  }
702  break;
703  }
704  }
705  }
706  // Commit to send the post-optimisation network structure
707  timelineUtils->Commit();
708 }

References ARMNN_SCOPED_PROFILING_EVENT, armnn::Input, armnn::Output, and armnn::Undefined.


The documentation for this class was generated from the following files:
armnn::ImportedInputId
unsigned int ImportedInputId
Definition: Types.hpp:310
armnn::Compute::Undefined
@ Undefined
armnn::INetworkProperties::m_AsyncEnabled
const bool m_AsyncEnabled
Definition: IRuntime.hpp:59
armnn::INetworkProperties::m_InputSource
const MemorySource m_InputSource
Definition: IRuntime.hpp:65
armnn::TensorHandleFactoryRegistry::ReleaseMemory
void ReleaseMemory()
Release memory required for inference.
Definition: TensorHandleFactoryRegistry.cpp:86
armnn::TensorHandleFactoryRegistry::GetFactory
ITensorHandleFactory * GetFactory(ITensorHandleFactory::FactoryId id) const
Find a TensorHandleFactory by Id Returns nullptr if not found.
Definition: TensorHandleFactoryRegistry.cpp:39
ARMNN_NO_DEPRECATE_WARN_BEGIN
#define ARMNN_NO_DEPRECATE_WARN_BEGIN
Definition: Deprecated.hpp:33
armnn::BoostLogSeverityMapping::error
@ error
armnn::ITensorHandleFactory::LegacyFactoryId
static const FactoryId LegacyFactoryId
Definition: ITensorHandleFactory.hpp:50
ARMNN_LOG
#define ARMNN_LOG(severity)
Definition: Logging.hpp:212
ARMNN_SCOPED_PROFILING_EVENT
#define ARMNN_SCOPED_PROFILING_EVENT(backendId, name)
Definition: Profiling.hpp:220
armnn::MemorySource::Undefined
@ Undefined
armnn::LayerBindingId
int LayerBindingId
Type of identifiers for bindable layers (inputs, outputs).
Definition: Types.hpp:309
armnn::CopyToOutputTensor
void CopyToOutputTensor(const Tensor &outputTensor, ITensorHandle *outputTensorHandle)
Definition: LoadedNetwork.cpp:1444
ARMNN_SCOPED_HEAP_PROFILING
#define ARMNN_SCOPED_HEAP_PROFILING(TAG)
Definition: HeapProfiling.hpp:45
armnn::CheckFlag
bool CheckFlag(MemorySourceFlags flags, MemorySource source)
Definition: MemorySources.hpp:41
armnn::Status::Success
@ Success
armnn::Exception
Base class for all ArmNN exceptions so that users can filter to just those.
Definition: Exceptions.hpp:46
armnn::RuntimeException
Definition: Exceptions.hpp:120
armnn::BoostLogSeverityMapping::info
@ info
armnn::LayerType::MemImport
@ MemImport
armnn::ImportedOutputId
unsigned int ImportedOutputId
Definition: Types.hpp:311
ARMNN_NO_DEPRECATE_WARN_END
#define ARMNN_NO_DEPRECATE_WARN_END
Definition: Deprecated.hpp:34
armnn::LoadedNetwork::Execute
Status Execute(const InputTensors &inputTensors, const OutputTensors &outputTensors, IWorkingMemHandle &workingMemHandle, std::vector< ImportedInputId > preImportedInputs={}, std::vector< ImportedOutputId > preImportedOutputs={})
Thread safe execution of the loaded network.
Definition: LoadedNetwork.cpp:1803
armnn::MemorySource
MemorySource
Define the Memory Source to reduce copies.
Definition: Types.hpp:244
armnn::GraphValidationException
Definition: Exceptions.hpp:110
armnn::LoadedNetwork::FreeWorkingMemory
void FreeWorkingMemory()
Definition: LoadedNetwork.cpp:1286
armnn::ITensorHandleFactory::FactoryId
std::string FactoryId
Definition: ITensorHandleFactory.hpp:49
armnn::LayerType::Input
@ Input
armnn::NullPointerException
Definition: Exceptions.hpp:146
armnn::INetworkProperties::m_OutputSource
const MemorySource m_OutputSource
Definition: IRuntime.hpp:66
armnn::Status::Failure
@ Failure
armnn::LayerType::Output
@ Output
armnn::LayerType::Constant
@ Constant
armnn::HasMatchingCapability
bool HasMatchingCapability(const BackendOptions::BackendOption &capability, const BackendCapabilities &capabilities)
Convenience function to check if a given capability matches a capability in a BackendCapabilities str...
Definition: BackendHelper.cpp:85