ArmNN
 24.08
IRuntime.hpp
Go to the documentation of this file.
1 //
2 // Copyright © 2017-2024 Arm Ltd and Contributors. All rights reserved.
3 // SPDX-License-Identifier: MIT
4 //
5 #pragma once
6 
7 #include "BackendOptions.hpp"
8 #include "INetwork.hpp"
9 #include "IProfiler.hpp"
10 #include "IWorkingMemHandle.hpp"
12 #include "Tensor.hpp"
13 #include "Types.hpp"
14 #include "TypesUtils.hpp"
15 
18 
19 #include <memory>
20 #include <map>
21 
22 namespace arm
23 {
24 
25 namespace pipe
26 {
27 class ILocalPacketHandler;
28 using ILocalPacketHandlerSharedPtr = std::shared_ptr<ILocalPacketHandler>;
29 } // pipe
30 } // arm
31 
32 namespace armnn
33 {
34 
35 using NetworkId = int;
36 
38 
39 struct RuntimeImpl;
40 class IRuntime;
41 using IRuntimePtr = std::unique_ptr<IRuntime, void(*)(IRuntime* runtime)>;
42 
44 {
45  INetworkProperties(bool asyncEnabled,
46  MemorySource inputSource,
47  MemorySource outputSource,
48  bool profilingEnabled = false,
50  bool externalMemoryManagementEnabled = false)
51  : m_AsyncEnabled(asyncEnabled),
52  m_ProfilingEnabled(profilingEnabled),
53  m_OutputNetworkDetailsMethod(detailsMethod),
54  m_InputSource(inputSource),
55  m_OutputSource(outputSource),
56  m_ExternalMemoryManagementEnabled(externalMemoryManagementEnabled)
57  {}
58 
59  const bool m_AsyncEnabled;
60 
61  const bool m_ProfilingEnabled;
62 
64 
67 
69 
70  virtual ~INetworkProperties() {}
71 };
72 
73 using namespace armnn::experimental;
74 
75 class IRuntime
76 {
77 public:
79  {
81  : m_GpuAccTunedParameters(nullptr)
82  , m_EnableGpuProfiling(false)
83  , m_DynamicBackendsPath("")
84  , m_ProtectedMode(false)
85  , m_CustomAllocatorMap()
86  , m_MemoryOptimizerStrategyMap()
87  {}
88 
89  /// If set, uses the GpuAcc tuned parameters from the given object when executing GPU workloads.
90  /// It will also be updated with new tuned parameters if it is configured to do so.
91  std::shared_ptr<IGpuAccTunedParameters> m_GpuAccTunedParameters;
92 
93  /// Setting this flag will allow the user to obtain GPU profiling information from the runtime.
95 
96  /// Setting this value will override the paths set by the DYNAMIC_BACKEND_PATHS compiler directive
97  /// Only a single path is allowed for the override
98  /// It defines the path to search for any [dynamic backend libraries](src/dynamic/README.md).
99  std::string m_DynamicBackendsPath;
100 
101  /// Setting this flag will allow the user to create the Runtime in protected mode.
102  /// It will run all the inferences on protected memory and will make sure that
103  /// INetworkProperties::m_ImportEnabled set to true with MemorySource::DmaBufProtected option
104  /// This requires that the backend supports Protected Memory and has an allocator capable of
105  /// allocating Protected Memory associated with it.
107 
108  /// @brief A map to define a custom memory allocator for specific backend Ids.
109  ///
110  /// @details A Custom Allocator is used for allocation of working memory in the backends.
111  /// Set this if you need to take control of how memory is allocated on a backend. Required for
112  /// Protected Mode in order to correctly allocate Protected Memory
113  ///
114  /// @note Only supported for GpuAcc
115  std::map<BackendId, std::shared_ptr<ICustomAllocator>> m_CustomAllocatorMap;
116 
117  /// @brief A map to define a custom memory optimizer strategy for specific backend Ids.
118  ///
119  /// @details A Memory Optimizer Strategy provides a solution to an abstract representation of
120  /// a network's memory requirements. This can also be used to return a pre-computed solution
121  /// for a specific network. Set this if you want to implement a Custom Memory Optimizer Strategy
122  /// for a given backend.
123  std::map<BackendId, std::shared_ptr<IMemoryOptimizerStrategy>> m_MemoryOptimizerStrategyMap;
124 
126  {
128  : m_EnableProfiling(false)
129  , m_TimelineEnabled(false)
130  , m_OutgoingCaptureFile("")
131  , m_IncomingCaptureFile("")
132  , m_FileOnly(false)
133  , m_CapturePeriod(LOWEST_CAPTURE_PERIOD)
134  , m_FileFormat("binary")
135  , m_LocalPacketHandlers()
136  {}
137 
138  /// Indicates whether external profiling is enabled or not.
140  /// Indicates whether external timeline profiling is enabled or not.
142  /// Path to a file in which outgoing timeline profiling messages will be stored.
144  /// Path to a file in which incoming timeline profiling messages will be stored.
146  /// Enable profiling output to file only.
148  /// The duration at which captured profiling messages will be flushed.
149  uint32_t m_CapturePeriod;
150  /// The format of the file used for outputting profiling data.
151  std::string m_FileFormat;
152  std::vector<arm::pipe::ILocalPacketHandlerSharedPtr> m_LocalPacketHandlers;
153  };
155 
156  /// Pass backend specific options.
157  ///
158  /// For example, to enable GpuAcc tuning add the following
159  /// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~.cpp
160  /// m_BackendOption.emplace_back(
161  /// BackendOptions{"GpuAcc",
162  /// {
163  /// {"TuningLevel", 2},
164  /// {"TuningFile", filename}
165  /// {"MemoryOptimizerStrategy", strategyname}
166  /// }
167  /// });
168  /// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
169  /// Execute representative workloads through the runtime to generate tuning data.
170  /// The tuning file is written once the runtime is destroyed
171 
172  /// To execute with the tuning data, start up with just the tuning file specified.
173  /// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~.cpp
174  /// m_BackendOption.emplace_back(
175  /// BackendOptions{"GpuAcc",
176  /// {
177  /// {"TuningFile", filename}
178  /// }
179  /// });
180  /// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
181 
182  /// The following backend options are available:
183  /// AllBackends:
184  /// "MemoryOptimizerStrategy" : string [stategynameString]
185  /// (Existing Memory Optimizer Strategies: ConstantMemoryStrategy)
186  /// GpuAcc:
187  /// "TuningLevel" : int [0..3] (0=UseOnly(default) | 1=RapidTuning | 2=NormalTuning | 3=ExhaustiveTuning)
188  /// "TuningFile" : string [filenameString]
189  /// "KernelProfilingEnabled" : bool [true | false]
190  std::vector<BackendOptions> m_BackendOptions;
191  };
192 
193  static IRuntime* CreateRaw(const CreationOptions& options);
194  static IRuntimePtr Create(const CreationOptions& options);
195  static void Destroy(IRuntime* runtime);
196 
197  /// Loads a complete network into the IRuntime.
198  /// @param [out] networkIdOut - Unique identifier for the network is returned in this reference.
199  /// @param [in] network - Complete network to load into the IRuntime.
200  /// The runtime takes ownership of the network once passed in.
201  /// @return armnn::Status
202  Status LoadNetwork(NetworkId& networkIdOut, IOptimizedNetworkPtr network);
203 
204  /// Load a complete network into the IRuntime.
205  /// @param [out] networkIdOut Unique identifier for the network is returned in this reference.
206  /// @param [in] network Complete network to load into the IRuntime.
207  /// @param [out] errorMessage Error message if there were any errors.
208  /// The runtime takes ownership of the network once passed in.
209  /// @return armnn::Status
210  Status LoadNetwork(NetworkId& networkIdOut,
211  IOptimizedNetworkPtr network,
212  std::string& errorMessage);
213 
214  Status LoadNetwork(NetworkId& networkIdOut,
215  IOptimizedNetworkPtr network,
216  std::string& errorMessage,
217  const INetworkProperties& networkProperties);
218 
219  TensorInfo GetInputTensorInfo(NetworkId networkId, LayerBindingId layerId) const;
220  TensorInfo GetOutputTensorInfo(NetworkId networkId, LayerBindingId layerId) const;
221 
222  /// ImportInputs separates the importing and mapping of InputTensors from network execution.
223  /// Allowing for a set of InputTensors to be imported and mapped once, but used in execution many times.
224  /// This function is not thread safe and must not be used while other threads are calling Execute().
225  /// No exceptions are thrown for failed imports. It is the caller's responsibility to check whether
226  /// tensors have been successfully imported by comparing returned ids with those passed in the InputTensors.
227  /// Whether a tensor can be imported or not is backend specific.
228  std::vector<ImportedInputId> ImportInputs(NetworkId networkId, const InputTensors& inputTensors,
229  MemorySource forceImportMemorySource = MemorySource::Undefined);
230 
231  /// ImportOutputs separates the importing and mapping of OutputTensors from network execution.
232  /// Allowing for a set of OutputTensors to be imported and mapped once, but used in execution many times.
233  /// This function is not thread safe and must not be used while other threads are calling Execute().
234  /// No exceptions are thrown for failed imports. It is the caller's responsibility to check whether
235  /// tensors have been successfully imported by comparing returned ids with those passed in the OutputTensors.
236  /// Whether a tensor can be imported or not is backend specific.
237  std::vector<ImportedOutputId> ImportOutputs(NetworkId networkId, const OutputTensors& outputTensors,
238  MemorySource forceImportMemorySource = MemorySource::Undefined);
239 
240  /// Un-import and delete the imported InputTensor/s
241  /// This function is not thread safe and must not be used while other threads are calling Execute().
242  /// Only compatible with AsyncEnabled networks
243  void ClearImportedInputs(NetworkId networkId, const std::vector<ImportedInputId> inputIds);
244 
245  /// Un-import and delete the imported OutputTensor/s
246  /// This function is not thread safe and must not be used while other threads are calling Execute().
247  /// Only compatible with AsyncEnabled networks
248  void ClearImportedOutputs(NetworkId networkId, const std::vector<ImportedOutputId> outputIds);
249 
250  /// Evaluates a network using input in inputTensors and outputs filled into outputTensors
251  Status EnqueueWorkload(NetworkId networkId,
252  const InputTensors& inputTensors,
253  const OutputTensors& outputTensors,
254  std::vector<ImportedInputId> preImportedInputIds = {},
255  std::vector<ImportedOutputId> preImportedOutputIds = {});
256 
257  /// This is an experimental function.
258  /// Evaluates a network using input in inputTensors and outputs filled into outputTensors.
259  /// This function performs a thread safe execution of the network. Returns once execution is complete.
260  /// Will block until this and any other thread using the same workingMem object completes.
261  ARMNN_DEPRECATED_MSG_REMOVAL_DATE("The Async interface will be removed from Arm NN in 24.08", "24.08")
262  Status Execute(IWorkingMemHandle& workingMemHandle,
263  const InputTensors& inputTensors,
264  const OutputTensors& outputTensors,
265  std::vector<ImportedInputId> preImportedInputs = {},
266  std::vector<ImportedOutputId> preImportedOutputs = {});
267 
268  /// Unloads a network from the IRuntime.
269  /// At the moment this only removes the network from the m_Impl->m_Network.
270  /// This might need more work in the future to be AndroidNN compliant.
271  /// @param [in] networkId - Unique identifier for the network to be unloaded. Generated in LoadNetwork().
272  /// @return armnn::Status
273  Status UnloadNetwork(NetworkId networkId);
274 
275  const IDeviceSpec& GetDeviceSpec() const;
276 
277  /// Create a new unique WorkingMemHandle object. Create multiple handles if you wish to have
278  /// overlapped Execution by calling this function from different threads.
279  std::unique_ptr<IWorkingMemHandle> CreateWorkingMemHandle(NetworkId networkId);
280 
281  /// Gets the profiler corresponding to the given network id.
282  /// @param networkId The id of the network for which to get the profile.
283  /// @return A pointer to the requested profiler, or nullptr if not found.
284  const std::shared_ptr<IProfiler> GetProfiler(NetworkId networkId) const;
285 
286  /// Registers a callback function to debug layers performing custom computations on intermediate tensors.
287  /// @param networkId The id of the network to register the callback.
288  /// @param func callback function to pass to the debug layer.
289  void RegisterDebugCallback(NetworkId networkId, const DebugCallbackFunction& func);
290 
291 protected:
292  IRuntime();
293  IRuntime(const IRuntime::CreationOptions& options);
294  ~IRuntime();
295 
296  std::unique_ptr<RuntimeImpl> pRuntimeImpl;
297 };
298 
299 
300 /// The following API is replaced by the backend options API.
301 using IGpuAccTunedParametersPtr = std::shared_ptr<IGpuAccTunedParameters>;
302 
303 /// Manages a set of GpuAcc parameters which have been tuned for maximum performance.
304 /// Passes an instance of this object to the IRuntime::Create() method (via IRuntime::CreationOptions) to use it
305 /// for all GPU workload execution.
306 ///
307 /// Can be created in two modes:
308 /// - In UseTunedParameters mode, the parameters stored in this object are used to execute GPU workloads.
309 /// - In UpdateTunedParameters mode, additionally, whenever a GPU workload is executed for the first time, the
310 /// optimum parameters will be found and stored in this object. WARNING - This tuning can be slow.
311 ///
312 /// The parameters can be loaded from and saved to a file so that you can first run a slow initial read-write
313 /// execution, save the parameters for later and then run fast read-only executions using the optimised parameters.
315 {
316 public:
317  enum class Mode
318  {
319  UseTunedParameters,
320  UpdateTunedParameters
321  };
322 
323  enum class TuningLevel
324  {
325  Rapid = 1,
326  Normal = 2,
327  Exhaustive = 3
328  };
329 
330  /// Creates an IClTunedParameters with the given mode.
331  /// @{
332  static IGpuAccTunedParameters* CreateRaw(Mode mode, TuningLevel tunerMode);
333  static IGpuAccTunedParametersPtr Create(Mode mode, TuningLevel tunerMode);
334  /// @}
335  static void Destroy(IGpuAccTunedParameters* params);
336 
337  /// Loads an existing set of tuned parameters from the given file.
338  /// If there is an error loading the file, an armnn::Exception is thrown.
339  virtual void Load(const char* filename) = 0;
340 
341  /// Saves the current set of tuned parameters to the given file.
342  /// If there is an error saving to the file, an armnn::Exception is thrown.
343  virtual void Save(const char* filename) const = 0;
344 
345 protected:
347 };
348 
349 } // namespace armnn
armnn::TuningLevel::Exhaustive
@ Exhaustive
armnn::IGpuAccTunedParameters::Mode
Mode
Definition: IRuntime.hpp:317
armnn::INetworkProperties::~INetworkProperties
virtual ~INetworkProperties()
Definition: IRuntime.hpp:70
armnn::ImportedInputId
unsigned int ImportedInputId
Definition: Types.hpp:310
armnn::IRuntime::CreationOptions::ExternalProfilingOptions::m_LocalPacketHandlers
std::vector< arm::pipe::ILocalPacketHandlerSharedPtr > m_LocalPacketHandlers
Definition: IRuntime.hpp:152
armnn::IOptimizedNetworkPtr
std::unique_ptr< IOptimizedNetwork, void(*)(IOptimizedNetwork *network)> IOptimizedNetworkPtr
Definition: INetwork.hpp:340
armnn::INetworkProperties::m_AsyncEnabled
const bool m_AsyncEnabled
Definition: IRuntime.hpp:59
arm
Definition: BackendRegistry.hpp:15
armnn::IRuntime::CreationOptions::ExternalProfilingOptions::m_FileOnly
bool m_FileOnly
Enable profiling output to file only.
Definition: IRuntime.hpp:147
armnn::IGpuAccTunedParameters::~IGpuAccTunedParameters
virtual ~IGpuAccTunedParameters()
Definition: IRuntime.hpp:346
armnn::INetworkProperties::m_InputSource
const MemorySource m_InputSource
Definition: IRuntime.hpp:65
IMemoryOptimizerStrategy.hpp
armnn::InputTensors
std::vector< std::pair< LayerBindingId, class ConstTensor > > InputTensors
Definition: Tensor.hpp:394
armnn::IRuntime::CreationOptions::CreationOptions
CreationOptions()
Definition: IRuntime.hpp:80
TypesUtils.hpp
armnn::IRuntime::CreationOptions::ExternalProfilingOptions
Definition: IRuntime.hpp:125
armnn::IRuntime::CreationOptions::m_DynamicBackendsPath
std::string m_DynamicBackendsPath
Setting this value will override the paths set by the DYNAMIC_BACKEND_PATHS compiler directive Only a...
Definition: IRuntime.hpp:99
IAsyncExecutionCallback.hpp
armnn::TensorInfo
Definition: Tensor.hpp:152
armnn::INetworkProperties::m_ProfilingEnabled
const bool m_ProfilingEnabled
Definition: IRuntime.hpp:61
armnn::INetworkProperties::m_ExternalMemoryManagementEnabled
const bool m_ExternalMemoryManagementEnabled
Definition: IRuntime.hpp:68
armnn::IGpuAccTunedParameters::TuningLevel
TuningLevel
Definition: IRuntime.hpp:323
ICustomAllocator.hpp
armnn::OutputTensors
std::vector< std::pair< LayerBindingId, class Tensor > > OutputTensors
Definition: Tensor.hpp:395
armnn::IRuntime::CreationOptions::ExternalProfilingOptions::m_IncomingCaptureFile
std::string m_IncomingCaptureFile
Path to a file in which incoming timeline profiling messages will be stored.
Definition: IRuntime.hpp:145
armnn::experimental::IWorkingMemHandle
Definition: IWorkingMemHandle.hpp:20
armnn::IRuntime
Definition: IRuntime.hpp:75
armnn::LOWEST_CAPTURE_PERIOD
constexpr unsigned int LOWEST_CAPTURE_PERIOD
The lowest performance data capture interval we support is 10 miliseconds.
Definition: Types.hpp:34
armnn::ProfilingDetailsMethod::Undefined
@ Undefined
armnn::IRuntime::CreationOptions::ExternalProfilingOptions::m_TimelineEnabled
bool m_TimelineEnabled
Indicates whether external timeline profiling is enabled or not.
Definition: IRuntime.hpp:141
armnn::IRuntime::CreationOptions::ExternalProfilingOptions::m_CapturePeriod
uint32_t m_CapturePeriod
The duration at which captured profiling messages will be flushed.
Definition: IRuntime.hpp:149
armnn::INetworkProperties::m_OutputNetworkDetailsMethod
const ProfilingDetailsMethod m_OutputNetworkDetailsMethod
Definition: IRuntime.hpp:63
armnn::IRuntime::CreationOptions::ExternalProfilingOptions::m_FileFormat
std::string m_FileFormat
The format of the file used for outputting profiling data.
Definition: IRuntime.hpp:151
armnn::NetworkId
int NetworkId
Definition: IRuntime.hpp:35
INetwork.hpp
armnn::IRuntime::CreationOptions::m_BackendOptions
std::vector< BackendOptions > m_BackendOptions
Pass backend specific options.
Definition: IRuntime.hpp:190
armnn::IRuntimePtr
std::unique_ptr< IRuntime, void(*)(IRuntime *runtime)> IRuntimePtr
Definition: IRuntime.hpp:41
armnn::MemorySource::Undefined
@ Undefined
armnn::experimental
Definition: IWorkload.hpp:13
armnn::LayerBindingId
int LayerBindingId
Type of identifiers for bindable layers (inputs, outputs).
Definition: Types.hpp:309
IProfiler.hpp
armnn::ProfilingDetailsMethod
ProfilingDetailsMethod
Define the behaviour of the internal profiler when outputting network details.
Definition: Types.hpp:71
arm::pipe::ILocalPacketHandlerSharedPtr
std::shared_ptr< ILocalPacketHandler > ILocalPacketHandlerSharedPtr
Definition: IRuntime.hpp:28
armnn::INetworkProperties
Definition: IRuntime.hpp:43
armnn::IRuntime::CreationOptions::ExternalProfilingOptions::ExternalProfilingOptions
ExternalProfilingOptions()
Definition: IRuntime.hpp:127
armnn::TuningLevel::Rapid
@ Rapid
armnn::IRuntime::CreationOptions::m_ProtectedMode
bool m_ProtectedMode
Setting this flag will allow the user to create the Runtime in protected mode.
Definition: IRuntime.hpp:106
armnn::TuningLevel::Normal
@ Normal
armnn::IRuntime::CreationOptions::m_ProfilingOptions
ExternalProfilingOptions m_ProfilingOptions
Definition: IRuntime.hpp:154
Tensor.hpp
armnn::Status
Status
Definition: Types.hpp:42
armnn::IRuntime::CreationOptions
Definition: IRuntime.hpp:78
std
Definition: BackendId.hpp:149
ARMNN_DEPRECATED_MSG_REMOVAL_DATE
#define ARMNN_DEPRECATED_MSG_REMOVAL_DATE(message, removed_in_release)
Definition: Deprecated.hpp:44
armnn::DebugCallbackFunction
std::function< void(LayerGuid guid, unsigned int slotIndex, ITensorHandle *tensorHandle)> DebugCallbackFunction
Define the type of callback for the Debug layer to call.
Definition: Types.hpp:398
armnn::MemorySource
MemorySource
Define the Memory Source to reduce copies.
Definition: Types.hpp:244
armnn::INetworkProperties::INetworkProperties
INetworkProperties(bool asyncEnabled, MemorySource inputSource, MemorySource outputSource, bool profilingEnabled=false, ProfilingDetailsMethod detailsMethod=ProfilingDetailsMethod::Undefined, bool externalMemoryManagementEnabled=false)
Definition: IRuntime.hpp:45
BackendOptions.hpp
armnn::RuntimeImpl
Definition: Runtime.hpp:30
armnn::IRuntime::CreationOptions::m_MemoryOptimizerStrategyMap
std::map< BackendId, std::shared_ptr< IMemoryOptimizerStrategy > > m_MemoryOptimizerStrategyMap
A map to define a custom memory optimizer strategy for specific backend Ids.
Definition: IRuntime.hpp:123
armnn
Copyright (c) 2021 ARM Limited and Contributors.
Definition: 01_00_quick_start.dox:6
Types.hpp
armnn::IRuntime::CreationOptions::ExternalProfilingOptions::m_OutgoingCaptureFile
std::string m_OutgoingCaptureFile
Path to a file in which outgoing timeline profiling messages will be stored.
Definition: IRuntime.hpp:143
armnn::IGpuAccTunedParametersPtr
std::shared_ptr< IGpuAccTunedParameters > IGpuAccTunedParametersPtr
The following API is replaced by the backend options API.
Definition: IRuntime.hpp:301
armnn::TuningLevel
TuningLevel
Definition: ArmComputeTuningUtils.hpp:18
armnn::IRuntime::CreationOptions::m_GpuAccTunedParameters
std::shared_ptr< IGpuAccTunedParameters > m_GpuAccTunedParameters
If set, uses the GpuAcc tuned parameters from the given object when executing GPU workloads.
Definition: IRuntime.hpp:91
armnn::IRuntime::CreationOptions::m_CustomAllocatorMap
std::map< BackendId, std::shared_ptr< ICustomAllocator > > m_CustomAllocatorMap
A map to define a custom memory allocator for specific backend Ids.
Definition: IRuntime.hpp:115
armnn::IRuntime::pRuntimeImpl
std::unique_ptr< RuntimeImpl > pRuntimeImpl
Definition: IRuntime.hpp:296
armnn::INetworkProperties::m_OutputSource
const MemorySource m_OutputSource
Definition: IRuntime.hpp:66
armnn::IRuntime::CreationOptions::ExternalProfilingOptions::m_EnableProfiling
bool m_EnableProfiling
Indicates whether external profiling is enabled or not.
Definition: IRuntime.hpp:139
armnn::IGpuAccTunedParameters
Manages a set of GpuAcc parameters which have been tuned for maximum performance.
Definition: IRuntime.hpp:314
armnn::IRuntime::CreationOptions::m_EnableGpuProfiling
bool m_EnableGpuProfiling
Setting this flag will allow the user to obtain GPU profiling information from the runtime.
Definition: IRuntime.hpp:94
IWorkingMemHandle.hpp