ArmNN
 24.08
IBackendInternal.hpp
Go to the documentation of this file.
1 //
2 // Copyright © 2022 Arm Ltd and Contributors. All rights reserved.
3 // SPDX-License-Identifier: MIT
4 //
5 
6 #pragma once
7 
8 #include <armnn/Types.hpp>
9 #include <armnn/IRuntime.hpp>
10 #include <armnn/Deprecated.hpp>
11 
12 #include <ExecutionData.hpp>
14 #include <WorkingMemDescriptor.hpp>
15 
21 
22 #include <client/include/backends/IBackendProfiling.hpp>
23 #include <client/include/backends/IBackendProfilingContext.hpp>
24 
25 #include <vector>
26 #include <memory>
27 
28 namespace armnn
29 {
30 class IWorkloadFactory;
31 class IMemoryManager;
32 class ILayerSupport;
33 
35 {
36  uint32_t m_Major;
37  uint32_t m_Minor;
38 
39  constexpr BackendVersion()
40  : m_Major(0)
41  , m_Minor(0)
42  {}
43  constexpr BackendVersion(uint32_t major, uint32_t minor)
44  : m_Major(major)
45  , m_Minor(minor)
46  {}
47 
48  bool operator==(const BackendVersion& other) const
49  {
50  return this == &other ||
51  (this->m_Major == other.m_Major &&
52  this->m_Minor == other.m_Minor);
53  }
54 
55  bool operator<=(const BackendVersion& other) const
56  {
57  return this->m_Major < other.m_Major ||
58  (this->m_Major == other.m_Major &&
59  this->m_Minor <= other.m_Minor);
60  }
61 
62  bool operator>=(const BackendVersion& other) const
63  {
64  return this->m_Major > other.m_Major ||
65  (this->m_Major == other.m_Major &&
66  this->m_Minor >= other.m_Minor);
67  }
68 };
69 
70 inline std::ostream& operator<<(std::ostream& os, const BackendVersion& backendVersion)
71 {
72  os << "[" << backendVersion.m_Major << "." << backendVersion.m_Minor << "]";
73 
74  return os;
75 }
76 
77 class IBackendInternal : public IBackend
78 {
79 protected:
80  /// Creation must be done through a specific
81  /// backend interface.
82  IBackendInternal() = default;
83 
84 public:
85  /// Allow backends created by the factory function
86  /// to be destroyed through IBackendInternal.
87  ~IBackendInternal() override = default;
88 
89  using IWorkloadFactoryPtr = std::unique_ptr<IWorkloadFactory>;
90  using IBackendContextPtr = std::unique_ptr<IBackendContext>;
91  /// This is the bridge between backend and backend profiling we'll keep it in the backend namespace.
92  using IBackendProfilingContextPtr = std::shared_ptr<arm::pipe::IBackendProfilingContext>;
93  using IBackendProfilingPtr = std::unique_ptr<arm::pipe::IBackendProfiling>;
94  using ILayerSupportSharedPtr = std::shared_ptr<ILayerSupport>;
95 
96  using IBackendSpecificModelContextPtr = std::shared_ptr<IBackendModelContext>;
97 
98  using IMemoryManagerUniquePtr = std::unique_ptr<IMemoryManager>;
99  using IMemoryManagerSharedPtr = std::shared_ptr<IMemoryManager>;
100 
102 
104  const IMemoryManagerSharedPtr& memoryManager = nullptr) const = 0;
105 
107  class TensorHandleFactoryRegistry& tensorHandleFactoryRegistry) const;
108 
110  const IMemoryManagerSharedPtr& memoryManager,
111  const ModelOptions& modelOptions) const;
112 
114  class TensorHandleFactoryRegistry& tensorHandleFactoryRegistry,
115  const ModelOptions& modelOptions) const;
116 
118  class TensorHandleFactoryRegistry& tensorHandleFactoryRegistry,
119  const ModelOptions& modelOptions,
120  MemorySourceFlags inputFlags,
121  MemorySourceFlags outputFlags) const;
122 
123  /// Create the runtime context of the backend
124  ///
125  /// Implementations may return a default-constructed IBackendContextPtr if
126  /// no context is needed at runtime.
127  /// Implementations must throw BackendUnavailableException if the backend
128  /// cannot be used (for example, necessary accelerator hardware is not present).
129  /// The default implementation always returns a default-constructed pointer.
131 
133 
134  /// Create context specifically used for profiling interaction from backends.
136  IBackendProfilingPtr& backendProfiling);
137 
138  virtual ILayerSupportSharedPtr GetLayerSupport() const = 0;
139 
140  virtual ILayerSupportSharedPtr GetLayerSupport(const ModelOptions& modelOptions) const;
141 
142  virtual OptimizationViews OptimizeSubgraphView(const SubgraphView& subgraph) const;
143 
144  virtual OptimizationViews OptimizeSubgraphView(const SubgraphView& subgraph,
145  const ModelOptions& modelOptions) const;
146 
147  bool SupportsTensorAllocatorAPI() const;
148 
150 
151  /// (Optional) Returns a vector of supported TensorHandleFactory ids in preference order.
152  virtual std::vector<ITensorHandleFactory::FactoryId> GetHandleFactoryPreferences() const;
153 
154  /// (Optional) Register TensorHandleFactories
155  /// Either this method or CreateMemoryManager() and
156  /// IWorkloadFactory::CreateTensor() IWorkloadFactory::CreateSubtensor() methods must be implemented.
157  virtual void RegisterTensorHandleFactories(class TensorHandleFactoryRegistry& /*registry*/) {}
158 
159  /// (Optional) Register TensorHandleFactories
160  /// Either this method or CreateMemoryManager() and
161  /// IWorkloadFactory::CreateTensor() IWorkloadFactory::CreateSubtensor() methods must be implemented.
162  virtual void RegisterTensorHandleFactories(class TensorHandleFactoryRegistry& registry,
163  MemorySourceFlags inputFlags,
164  MemorySourceFlags outputFlags);
165 
166  /// Returns the version of the Backend API
167  static constexpr BackendVersion GetApiVersion() { return BackendVersion(1, 0); }
168 
169  /// Returns a BackendCapability if the backend lists the capability
170  /// The BackendCapability must then be inspected to check whether or not that BackendCapability is supported
171  /// Otherwise returns an EmptyOptional if the BackendCapability is unlisted
173  {
174  return BackendCapabilities("IBackendInternal NullCapabilities");
175  };
176 
177  /// Signals the backend to use a custom memory allocator provided by the user
178  ///
179  /// \param allocator - a pointer to the provided ICustomAllocator to use with this backend
180  /// \param errMsg - Optional string variable to return error messages
181  /// \return - Returns true if switching to custom allocator was successful
182  virtual bool UseCustomMemoryAllocator(std::shared_ptr<ICustomAllocator> allocator,
184  {
185  IgnoreUnused(allocator);
186  if (errMsg)
187  {
188  std::stringstream message;
189  message << "The backend " << GetId() << " doesn't support using a custom allocator. This error might"
190  " be related with the protected mode if the backend doesn't"
191  " fully support it.";
192 
193  errMsg.value() = message.str();
194  }
195  return false;
196  }
197 
198  /// Returns the default memory allocator for the backend
199  ///
200  /// \return - Returns unique pointer to the Default Allocator of the Backend
201  virtual std::unique_ptr<ICustomAllocator> GetDefaultAllocator() const
202  {
203  throw armnn::Exception("GetDefaultAllocator: Function has not been implemented in backend.");
204  }
205 
206  /// Returns the number of files cached if backend supports caching
207  ///
208  /// \return - Returns 0 if backend does not support caching otherwise number of files cached
209  virtual unsigned int GetNumberOfCacheFiles() const { return 0; }
210 
211  /// Returns ExecutionData for the backend
212  ///
213  /// \param workingMemDescriptor - Vectors of input and output TensorHandles for a layer
214  /// \return - Returns backend specific ExecutionData generated for a layer
215  virtual ExecutionData CreateExecutionData(WorkingMemDescriptor& workingMemDescriptor) const
216  {
217  IgnoreUnused(workingMemDescriptor);
218  throw armnn::Exception("CreateExecutionData: Function has not been implemented in backend.");
219  };
220 
221  /// Update the ExecutionData for a layer. It is used to swap in pre-imported tensor handles
222  ///
223  /// \param executionData - Backend specific ExecutionData generated for a layer
224  /// \param workingMemDescriptor - Vectors of input and output TensorHandles for a layer
225  virtual void UpdateExecutionData(ExecutionData& executionData, WorkingMemDescriptor& workingMemDescriptor) const
226  {
227  IgnoreUnused(executionData);
228  IgnoreUnused(workingMemDescriptor);
229  throw armnn::Exception("UpdateExecutionData: Function has not been implemented in backend.");
230  };
231 };
232 
233 using IBackendInternalUniquePtr = std::unique_ptr<IBackendInternal>;
234 
235 } // namespace armnn
armnn::IBackendInternal::GetBackwardCompatibleFavoriteHandleFactory
ITensorHandleFactory::FactoryId GetBackwardCompatibleFavoriteHandleFactory()
Definition: IBackendInternal.cpp:132
armnn::BackendVersion::m_Minor
uint32_t m_Minor
Definition: IBackendInternal.hpp:37
IRuntime.hpp
armnn::Optional
Definition: Optional.hpp:270
ExecutionData.hpp
armnn::IBackendInternal::GetNumberOfCacheFiles
virtual unsigned int GetNumberOfCacheFiles() const
Returns the number of files cached if backend supports caching.
Definition: IBackendInternal.hpp:209
armnn::IBackendInternal::IMemoryManagerSharedPtr
std::shared_ptr< IMemoryManager > IMemoryManagerSharedPtr
Definition: IBackendInternal.hpp:99
armnn::TensorHandleFactoryRegistry
Definition: TensorHandleFactoryRegistry.hpp:23
armnn::BackendCapabilities
BackendOptions BackendCapabilities
Definition: BackendOptions.hpp:19
armnn::IBackendInternal::GetLayerSupport
virtual ILayerSupportSharedPtr GetLayerSupport() const =0
Deprecated.hpp
armnn::IBackendInternal::UpdateExecutionData
virtual void UpdateExecutionData(ExecutionData &executionData, WorkingMemDescriptor &workingMemDescriptor) const
Update the ExecutionData for a layer.
Definition: IBackendInternal.hpp:225
armnn::MemorySourceFlags
unsigned int MemorySourceFlags
Definition: MemorySources.hpp:15
ITensorHandleFactory.hpp
armnn::IBackendInternal::CreateWorkloadFactory
virtual IWorkloadFactoryPtr CreateWorkloadFactory(const IMemoryManagerSharedPtr &memoryManager=nullptr) const =0
armnn::IBackendInternal
Definition: IBackendInternal.hpp:77
armnn::IBackendInternal::IBackendContextPtr
std::unique_ptr< IBackendContext > IBackendContextPtr
Definition: IBackendInternal.hpp:90
OptimizationViews.hpp
armnn::BackendVersion::operator<=
bool operator<=(const BackendVersion &other) const
Definition: IBackendInternal.hpp:55
armnn::BackendVersion::BackendVersion
constexpr BackendVersion()
Definition: IBackendInternal.hpp:39
armnn::IBackendInternal::OptimizeSubgraphView
virtual OptimizationViews OptimizeSubgraphView(const SubgraphView &subgraph) const
Definition: IBackendInternal.cpp:106
ISubgraphViewConverter.hpp
armnn::IBackendInternal::IBackendInternal
IBackendInternal()=default
Creation must be done through a specific backend interface.
armnn::IBackendInternal::CreateExecutionData
virtual ExecutionData CreateExecutionData(WorkingMemDescriptor &workingMemDescriptor) const
Returns ExecutionData for the backend.
Definition: IBackendInternal.hpp:215
armnn::IBackend::GetId
virtual const BackendId & GetId() const =0
armnn::IBackendInternal::CreateBackendContext
virtual IBackendContextPtr CreateBackendContext(const IRuntime::CreationOptions &) const
Create the runtime context of the backend.
Definition: IBackendInternal.cpp:71
armnn::operator<<
std::ostream & operator<<(std::ostream &os, const std::vector< Compute > &compute)
Deprecated function that will be removed together with the Compute enum.
Definition: BackendId.hpp:47
IBackendContext.hpp
armnn::IBackendInternal::GetDefaultAllocator
virtual std::unique_ptr< ICustomAllocator > GetDefaultAllocator() const
Returns the default memory allocator for the backend.
Definition: IBackendInternal.hpp:201
armnn::BackendVersion::BackendVersion
constexpr BackendVersion(uint32_t major, uint32_t minor)
Definition: IBackendInternal.hpp:43
armnn::SubgraphView
The SubgraphView class represents a subgraph of a Graph.
Definition: SubgraphView.hpp:31
WorkingMemDescriptor.hpp
armnn::OptimizationViews
Definition: OptimizationViews.hpp:17
armnn::IBackendInternal::CreateMemoryManager
virtual IMemoryManagerUniquePtr CreateMemoryManager() const
Definition: IBackendInternal.cpp:13
armnn::IBackend
Each backend should implement an IBackend.
Definition: Types.hpp:272
armnn::Exception
Base class for all ArmNN exceptions so that users can filter to just those.
Definition: Exceptions.hpp:46
armnn::IBackendInternal::GetApiVersion
static constexpr BackendVersion GetApiVersion()
Returns the version of the Backend API.
Definition: IBackendInternal.hpp:167
armnn::IBackendInternal::IBackendProfilingContextPtr
std::shared_ptr< arm::pipe::IBackendProfilingContext > IBackendProfilingContextPtr
This is the bridge between backend and backend profiling we'll keep it in the backend namespace.
Definition: IBackendInternal.hpp:92
armnn::IBackendInternal::CreateBackendProfilingContext
virtual IBackendProfilingContextPtr CreateBackendProfilingContext(const IRuntime::CreationOptions &creationOptions, IBackendProfilingPtr &backendProfiling)
Create context specifically used for profiling interaction from backends.
Definition: IBackendInternal.cpp:82
armnn::IBackendInternal::RegisterTensorHandleFactories
virtual void RegisterTensorHandleFactories(class TensorHandleFactoryRegistry &)
(Optional) Register TensorHandleFactories Either this method or CreateMemoryManager() and IWorkloadFa...
Definition: IBackendInternal.hpp:157
armnn::IBackendInternal::CreateBackendSpecificModelContext
virtual IBackendSpecificModelContextPtr CreateBackendSpecificModelContext(const ModelOptions &modelOptions) const
Definition: IBackendInternal.cpp:76
armnn::IBackendInternal::~IBackendInternal
~IBackendInternal() override=default
Allow backends created by the factory function to be destroyed through IBackendInternal.
armnn::IBackendInternal::UseCustomMemoryAllocator
virtual bool UseCustomMemoryAllocator(std::shared_ptr< ICustomAllocator > allocator, armnn::Optional< std::string & > errMsg)
Signals the backend to use a custom memory allocator provided by the user.
Definition: IBackendInternal.hpp:182
SubgraphView.hpp
armnn::BackendOptions
Struct for the users to pass backend specific options.
Definition: BackendOptions.hpp:22
armnn::BackendVersion::operator>=
bool operator>=(const BackendVersion &other) const
Definition: IBackendInternal.hpp:62
armnn::IBackendInternal::IBackendProfilingPtr
std::unique_ptr< arm::pipe::IBackendProfiling > IBackendProfilingPtr
Definition: IBackendInternal.hpp:93
armnn::IRuntime::CreationOptions
Definition: IRuntime.hpp:78
armnn::IBackendInternal::SupportsTensorAllocatorAPI
bool SupportsTensorAllocatorAPI() const
Definition: IBackendInternal.cpp:120
armnn::IBackendInternal::IMemoryManagerUniquePtr
std::unique_ptr< IMemoryManager > IMemoryManagerUniquePtr
Definition: IBackendInternal.hpp:98
armnn::IBackendInternal::GetCapabilities
virtual BackendCapabilities GetCapabilities() const
Returns a BackendCapability if the backend lists the capability The BackendCapability must then be in...
Definition: IBackendInternal.hpp:172
armnn::IgnoreUnused
void IgnoreUnused(Ts &&...)
Definition: IgnoreUnused.hpp:14
armnn::BackendVersion
Definition: IBackendInternal.hpp:34
armnn
Copyright (c) 2021 ARM Limited and Contributors.
Definition: 01_00_quick_start.dox:6
Types.hpp
armnn::experimental::WorkingMemDescriptor
Definition: WorkingMemDescriptor.hpp:18
armnn::ITensorHandleFactory::FactoryId
std::string FactoryId
Definition: ITensorHandleFactory.hpp:49
armnn::IBackendInternal::ILayerSupportSharedPtr
std::shared_ptr< ILayerSupport > ILayerSupportSharedPtr
Definition: IBackendInternal.hpp:94
armnn::IBackendInternal::GetHandleFactoryPreferences
virtual std::vector< ITensorHandleFactory::FactoryId > GetHandleFactoryPreferences() const
(Optional) Returns a vector of supported TensorHandleFactory ids in preference order.
Definition: IBackendInternal.cpp:143
armnn::ModelOptions
std::vector< BackendOptions > ModelOptions
Definition: BackendOptions.hpp:18
armnn::BackendVersion::operator==
bool operator==(const BackendVersion &other) const
Definition: IBackendInternal.hpp:48
armnn::IBackendInternal::IWorkloadFactoryPtr
std::unique_ptr< IWorkloadFactory > IWorkloadFactoryPtr
Definition: IBackendInternal.hpp:89
armnn::BackendVersion::m_Major
uint32_t m_Major
Definition: IBackendInternal.hpp:36
armnn::IBackendInternalUniquePtr
std::unique_ptr< IBackendInternal > IBackendInternalUniquePtr
Definition: BackendRegistry.hpp:32
IMemoryManager.hpp
armnn::OptionalReferenceSwitch< std::is_reference< T >::value, T >::value
const T & value() const
Definition: Optional.hpp:146
armnn::IBackendInternal::IBackendSpecificModelContextPtr
std::shared_ptr< IBackendModelContext > IBackendSpecificModelContextPtr
Definition: IBackendInternal.hpp:96
armnn::experimental::ExecutionData
Definition: ExecutionData.hpp:14