ArmNN
 24.08
ClBackend.hpp
Go to the documentation of this file.
1 //
2 // Copyright © 2017-2023 Arm Ltd and Contributors. All rights reserved.
3 // SPDX-License-Identifier: MIT
4 //
5 #pragma once
6 
8 
9 #include <arm_compute/core/Types.h>
10 #include <arm_compute/runtime/CL/CLBufferAllocator.h>
11 
13 #include <arm_compute/runtime/CL/CLMemoryRegion.h>
14 
15 #include <arm_compute/core/CL/CLKernelLibrary.h>
16 #include <CL/cl_ext.h>
17 
18 // System includes for mapping and unmapping memory
19 #include <sys/mman.h>
20 
21 namespace armnn
22 {
23 
25 {
26 public:
27  ClBackend() : m_CustomAllocator(nullptr) {};
28  ClBackend(std::shared_ptr<ICustomAllocator> allocator)
29  {
30  std::string err;
31  UseCustomMemoryAllocator(allocator, err);
32  }
33  ~ClBackend() = default;
34 
35  static const BackendId& GetIdStatic();
36  const BackendId& GetId() const override { return GetIdStatic(); }
37 
39 
41  const IBackendInternal::IMemoryManagerSharedPtr& memoryManager = nullptr) const override;
42 
44  TensorHandleFactoryRegistry& registry) const override;
45 
47  const ModelOptions& modelOptions) const override;
48 
50  const ModelOptions& modelOptions) const override;
51 
53  const ModelOptions& modelOptions,
54  MemorySourceFlags inputFlags,
55  MemorySourceFlags outputFlags) const override;
56 
57  std::vector<ITensorHandleFactory::FactoryId> GetHandleFactoryPreferences() const override;
58 
60 
62  MemorySourceFlags inputFlags,
63  MemorySourceFlags outputFlags) override;
64 
67  const IRuntime::CreationOptions&, IBackendProfilingPtr& backendProfiling) override;
68 
70  IBackendInternal::ILayerSupportSharedPtr GetLayerSupport(const ModelOptions& modelOptions) const override;
71 
73  const ModelOptions& modelOptions) const override;
74 
76  const ModelOptions& modelOptions) const override;
77 
78  std::unique_ptr<ICustomAllocator> GetDefaultAllocator() const override;
79 
80  BackendCapabilities GetCapabilities() const override;
81 
82  virtual bool UseCustomMemoryAllocator(std::shared_ptr<ICustomAllocator> allocator,
83  armnn::Optional<std::string&> errMsg) override
84  {
85  IgnoreUnused(errMsg);
86  ARMNN_LOG(info) << "Using Custom Allocator for ClBackend";
87 
88  // Set flag to signal the backend to use a custom memory allocator
89  m_CustomAllocator = std::make_shared<ClBackendCustomAllocatorWrapper>(std::move(allocator));
92  }
93 
94  virtual unsigned int GetNumberOfCacheFiles() const override { return 1; }
95 
96  // Cl requires a arm_compute::IAllocator we wrap the Arm NN ICustomAllocator to achieve this
97  class ClBackendCustomAllocatorWrapper : public arm_compute::IAllocator
98  {
99  public:
100  ClBackendCustomAllocatorWrapper(std::shared_ptr<ICustomAllocator> alloc) : m_CustomAllocator(alloc)
101  {}
102  // Inherited methods overridden:
103  void* allocate(size_t size, size_t alignment) override
104  {
105  auto alloc = m_CustomAllocator->allocate(size, alignment);
106  return MapAllocatedMemory(alloc, size, m_CustomAllocator->GetMemorySourceType());
107  }
108  void free(void* ptr) override
109  {
110  auto hostMemPtr = m_AllocatedBufferMappings[ptr];
111  clReleaseMemObject(static_cast<cl_mem>(ptr));
112  m_CustomAllocator->free(hostMemPtr);
113  }
114  std::unique_ptr<arm_compute::IMemoryRegion> make_region(size_t size, size_t alignment) override
115  {
116  auto hostMemPtr = m_CustomAllocator->allocate(size, alignment);
117  cl_mem buffer = MapAllocatedMemory(hostMemPtr, size, m_CustomAllocator->GetMemorySourceType());
118 
119  return std::make_unique<ClBackendCustomAllocatorMemoryRegion>(cl::Buffer(buffer),
120  hostMemPtr,
121  m_CustomAllocator->GetMemorySourceType());
122  }
123  private:
124  cl_mem MapAllocatedMemory(void* memory, size_t size, MemorySource source)
125  {
126  // Round the size of the buffer to a multiple of the CL_DEVICE_GLOBAL_MEM_CACHELINE_SIZE
127  auto cachelineAlignment =
128  arm_compute::CLKernelLibrary::get().get_device().getInfo<CL_DEVICE_GLOBAL_MEM_CACHELINE_SIZE>();
129  auto roundedSize = cachelineAlignment + size - (size % cachelineAlignment);
130 
131  if (source == MemorySource::Malloc)
132  {
133  const cl_import_properties_arm importProperties[] =
134  {
135  CL_IMPORT_TYPE_ARM,
136  CL_IMPORT_TYPE_HOST_ARM,
137  0
138  };
139  cl_int error = CL_SUCCESS;
140  cl_mem buffer = clImportMemoryARM(arm_compute::CLKernelLibrary::get().context().get(),
141  CL_MEM_READ_WRITE,
142  importProperties,
143  memory,
144  roundedSize,
145  &error);
146  if (error == CL_SUCCESS)
147  {
148  m_AllocatedBufferMappings.insert(std::make_pair(static_cast<void *>(buffer), memory));
149  return buffer;
150  }
151  throw armnn::Exception(
152  "Mapping allocated memory from CustomMemoryAllocator failed, errcode: " + std::to_string(error));
153  }
154  else if (source == MemorySource::DmaBuf)
155  {
156  const cl_import_properties_arm importProperties[] =
157  {
158  CL_IMPORT_TYPE_ARM,
159  CL_IMPORT_TYPE_DMA_BUF_ARM,
160  CL_IMPORT_DMA_BUF_DATA_CONSISTENCY_WITH_HOST_ARM,
161  CL_TRUE,
162  0
163  };
164  cl_int error = CL_SUCCESS;
165  cl_mem buffer = clImportMemoryARM(arm_compute::CLKernelLibrary::get().context().get(),
166  CL_MEM_READ_WRITE,
167  importProperties,
168  memory,
169  roundedSize,
170  &error);
171  if (error == CL_SUCCESS)
172  {
173  m_AllocatedBufferMappings.insert(std::make_pair(static_cast<void *>(buffer), memory));
174  return buffer;
175  }
176  throw armnn::Exception(
177  "Mapping allocated memory from CustomMemoryAllocator failed, errcode: "
178  + std::to_string(error));
179  }
180  else if (source == MemorySource::DmaBufProtected)
181  {
182  const cl_import_properties_arm importProperties[] =
183  {
184  CL_IMPORT_TYPE_ARM,
185  CL_IMPORT_TYPE_DMA_BUF_ARM,
186  CL_IMPORT_TYPE_PROTECTED_ARM,
187  CL_TRUE,
188  0
189  };
190  cl_int error = CL_SUCCESS;
191  cl_mem buffer = clImportMemoryARM(arm_compute::CLKernelLibrary::get().context().get(),
192  CL_MEM_READ_WRITE,
193  importProperties,
194  memory,
195  roundedSize,
196  &error);
197  if (error == CL_SUCCESS)
198  {
199  m_AllocatedBufferMappings.insert(std::make_pair(static_cast<void *>(buffer), memory));
200  return buffer;
201  }
202  throw armnn::Exception(
203  "Mapping allocated memory from CustomMemoryAllocator failed, errcode: "
204  + std::to_string(error));
205  }
206  throw armnn::Exception(
207  "Attempting to allocate memory with unsupported MemorySource type in CustomAllocator");
208  }
209  std::shared_ptr<ICustomAllocator> m_CustomAllocator;
210  std::map<void*, void*> m_AllocatedBufferMappings;
211  };
212 
213  class ClBackendCustomAllocatorMemoryRegion : public arm_compute::ICLMemoryRegion
214  {
215  public:
216  // We need to have a new version of ICLMemoryRegion which holds a hostMemPtr to allow for cpu copy access
217  ClBackendCustomAllocatorMemoryRegion(const cl::Buffer &buffer, void* hostMemPtr, armnn::MemorySource source)
218  : ICLMemoryRegion(buffer.getInfo<CL_MEM_SIZE>())
219  {
220  _mem = buffer;
221  m_HostMemPtr = hostMemPtr;
222  m_MemorySource = source;
223  }
224 
225  // Inherited methods overridden :
226  void* ptr() override
227  {
228  return nullptr;
229  }
230 
231  void* map(cl::CommandQueue &q, bool blocking) override
232  {
233  armnn::IgnoreUnused(q, blocking);
234  if (m_HostMemPtr == nullptr)
235  {
236  throw armnn::Exception("ClBackend: Attempting to map memory with an invalid host ptr");
237  }
238  if (_mapping != nullptr)
239  {
240  throw armnn::Exception("ClBackend: Attempting to map memory which has not yet been unmapped");
241  }
242  switch (m_MemorySource)
243  {
245  _mapping = m_HostMemPtr;
246  return _mapping;
247  break;
250  // If the source is a Dmabuf then the memory ptr should be pointing to an integer value for the fd
251  _mapping = mmap(NULL, _size, PROT_WRITE, MAP_SHARED, *(reinterpret_cast<int*>(m_HostMemPtr)), 0);
252  return _mapping;
253  break;
254  default:
255  throw armnn::Exception("ClBackend: Attempting to map imported memory without a valid source");
256  break;
257  }
258  }
259 
260  void unmap(cl::CommandQueue &q) override
261  {
263  switch (m_MemorySource)
264  {
266  _mapping = nullptr;
267  break;
270  munmap(_mapping, _size);
271  _mapping = nullptr;
272  break;
273  default:
274  throw armnn::Exception("ClBackend: Attempting to unmap imported memory without a valid source");
275  break;
276  }
277  }
278  private:
279  void* m_HostMemPtr = nullptr;
280  armnn::MemorySource m_MemorySource;
281  };
282 
283  std::shared_ptr<ClBackendCustomAllocatorWrapper> m_CustomAllocator;
285 };
286 
287 } // namespace armnn
armnn::MemorySource::Malloc
@ Malloc
armnn::ClBackend::ClBackend
ClBackend(std::shared_ptr< ICustomAllocator > allocator)
Definition: ClBackend.hpp:28
armnn::ClBackend::GetLayerSupport
IBackendInternal::ILayerSupportSharedPtr GetLayerSupport() const override
Definition: ClBackend.cpp:252
armnn::ClBackend::m_UsingCustomAllocator
bool m_UsingCustomAllocator
Definition: ClBackend.hpp:284
armnn::Optional
Definition: Optional.hpp:270
armnn::ClBackend::ClBackendCustomAllocatorMemoryRegion::map
void * map(cl::CommandQueue &q, bool blocking) override
Definition: ClBackend.hpp:231
armnn::ClBackend::RegisterTensorHandleFactories
void RegisterTensorHandleFactories(TensorHandleFactoryRegistry &registry) override
(Optional) Register TensorHandleFactories Either this method or CreateMemoryManager() and IWorkloadFa...
Definition: ClBackend.cpp:175
armnn::IBackendInternal::IMemoryManagerSharedPtr
std::shared_ptr< IMemoryManager > IMemoryManagerSharedPtr
Definition: IBackendInternal.hpp:99
armnn::TensorHandleFactoryRegistry
Definition: TensorHandleFactoryRegistry.hpp:23
armnn::MemorySource::DmaBufProtected
@ DmaBufProtected
armnn::MemorySourceFlags
unsigned int MemorySourceFlags
Definition: MemorySources.hpp:15
armnn::ClBackend::GetHandleFactoryPreferences
std::vector< ITensorHandleFactory::FactoryId > GetHandleFactoryPreferences() const override
(Optional) Returns a vector of supported TensorHandleFactory ids in preference order.
Definition: ClBackend.cpp:169
BaseMemoryManager.hpp
armnn::ClBackend::~ClBackend
~ClBackend()=default
armnn::IBackendInternal
Definition: IBackendInternal.hpp:77
armnn::IBackendInternal::IBackendContextPtr
std::unique_ptr< IBackendContext > IBackendContextPtr
Definition: IBackendInternal.hpp:90
armnn::BoostLogSeverityMapping::error
@ error
armnn::ClBackend::GetDefaultAllocator
std::unique_ptr< ICustomAllocator > GetDefaultAllocator() const override
Returns the default memory allocator for the backend.
Definition: ClBackend.cpp:270
armnn::ClBackend::UseCustomMemoryAllocator
virtual bool UseCustomMemoryAllocator(std::shared_ptr< ICustomAllocator > allocator, armnn::Optional< std::string & > errMsg) override
Signals the backend to use a custom memory allocator provided by the user.
Definition: ClBackend.hpp:82
armnn::ClBackend::ClBackendCustomAllocatorWrapper
Definition: ClBackend.hpp:97
armnn::ClBackend::CreateBackendProfilingContext
IBackendInternal::IBackendProfilingContextPtr CreateBackendProfilingContext(const IRuntime::CreationOptions &, IBackendProfilingPtr &backendProfiling) override
Create context specifically used for profiling interaction from backends.
Definition: ClBackend.cpp:240
ARMNN_LOG
#define ARMNN_LOG(severity)
Definition: Logging.hpp:212
armnn::ClBackend
Definition: ClBackend.hpp:24
armnn::ClBackend::ClBackendCustomAllocatorMemoryRegion::ClBackendCustomAllocatorMemoryRegion
ClBackendCustomAllocatorMemoryRegion(const cl::Buffer &buffer, void *hostMemPtr, armnn::MemorySource source)
Definition: ClBackend.hpp:217
armnn::ClBackend::GetCapabilities
BackendCapabilities GetCapabilities() const override
Returns a BackendCapability if the backend lists the capability The BackendCapability must then be in...
Definition: ClBackend.cpp:275
armnn::MemorySource::DmaBuf
@ DmaBuf
IBackendInternal.hpp
armnn::ClBackend::GetNumberOfCacheFiles
virtual unsigned int GetNumberOfCacheFiles() const override
Returns the number of files cached if backend supports caching.
Definition: ClBackend.hpp:94
armnn::ClBackend::m_CustomAllocator
std::shared_ptr< ClBackendCustomAllocatorWrapper > m_CustomAllocator
Definition: ClBackend.hpp:283
armnn::ClBackend::ClBackendCustomAllocatorWrapper::ClBackendCustomAllocatorWrapper
ClBackendCustomAllocatorWrapper(std::shared_ptr< ICustomAllocator > alloc)
Definition: ClBackend.hpp:100
armnn::SubgraphView
The SubgraphView class represents a subgraph of a Graph.
Definition: SubgraphView.hpp:31
armnn::OptimizationViews
Definition: OptimizationViews.hpp:17
armnn::Exception
Base class for all ArmNN exceptions so that users can filter to just those.
Definition: Exceptions.hpp:46
armnn::ClBackend::CreateBackendContext
IBackendInternal::IBackendContextPtr CreateBackendContext(const IRuntime::CreationOptions &) const override
Create the runtime context of the backend.
Definition: ClBackend.cpp:235
armnn::ClBackend::CreateWorkloadFactory
IBackendInternal::IWorkloadFactoryPtr CreateWorkloadFactory(const IBackendInternal::IMemoryManagerSharedPtr &memoryManager=nullptr) const override
Definition: ClBackend.cpp:59
armnn::BoostLogSeverityMapping::info
@ info
armnn::IBackendInternal::IBackendProfilingContextPtr
std::shared_ptr< arm::pipe::IBackendProfilingContext > IBackendProfilingContextPtr
This is the bridge between backend and backend profiling we'll keep it in the backend namespace.
Definition: IBackendInternal.hpp:92
armnn::ClBackend::OptimizeSubgraphView
OptimizationViews OptimizeSubgraphView(const SubgraphView &subgraph, const ModelOptions &modelOptions) const override
Definition: ClBackend.cpp:292
armnn::ClBackend::ClBackendCustomAllocatorMemoryRegion
Definition: ClBackend.hpp:213
armnn::BackendOptions
Struct for the users to pass backend specific options.
Definition: BackendOptions.hpp:22
armnn::IBackendInternal::IBackendProfilingPtr
std::unique_ptr< arm::pipe::IBackendProfiling > IBackendProfilingPtr
Definition: IBackendInternal.hpp:93
armnn::IRuntime::CreationOptions
Definition: IRuntime.hpp:78
armnn::ClBackend::ClBackendCustomAllocatorMemoryRegion::unmap
void unmap(cl::CommandQueue &q) override
Definition: ClBackend.hpp:260
armnn::IBackendInternal::IMemoryManagerUniquePtr
std::unique_ptr< IMemoryManager > IMemoryManagerUniquePtr
Definition: IBackendInternal.hpp:98
armnn::IgnoreUnused
void IgnoreUnused(Ts &&...)
Definition: IgnoreUnused.hpp:14
armnn::MemorySource
MemorySource
Define the Memory Source to reduce copies.
Definition: Types.hpp:244
armnn::BackendId
Definition: BackendId.hpp:75
armnn
Copyright (c) 2021 ARM Limited and Contributors.
Definition: 01_00_quick_start.dox:6
armnn::ClBackend::GetIdStatic
static const BackendId & GetIdStatic()
Definition: ClBackend.cpp:44
armnn::ClBackend::CreateBackendSpecificModelContext
IBackendInternal::IBackendSpecificModelContextPtr CreateBackendSpecificModelContext(const ModelOptions &modelOptions) const override
Definition: ClBackend.cpp:246
armnn::ClBackend::ClBackendCustomAllocatorWrapper::allocate
void * allocate(size_t size, size_t alignment) override
Definition: ClBackend.hpp:103
armnn::IBackendInternal::ILayerSupportSharedPtr
std::shared_ptr< ILayerSupport > ILayerSupportSharedPtr
Definition: IBackendInternal.hpp:94
armnn::ClBackend::ClBackendCustomAllocatorMemoryRegion::ptr
void * ptr() override
Definition: ClBackend.hpp:226
armnn::ModelOptions
std::vector< BackendOptions > ModelOptions
Definition: BackendOptions.hpp:18
armnn::ClBackend::ClBackendCustomAllocatorWrapper::make_region
std::unique_ptr< arm_compute::IMemoryRegion > make_region(size_t size, size_t alignment) override
Definition: ClBackend.hpp:114
armnn::IBackendInternal::IWorkloadFactoryPtr
std::unique_ptr< IWorkloadFactory > IWorkloadFactoryPtr
Definition: IBackendInternal.hpp:89
armnn::ClBackend::ClBackendCustomAllocatorWrapper::free
void free(void *ptr) override
Definition: ClBackend.hpp:108
armnn::ClBackend::GetId
const BackendId & GetId() const override
Definition: ClBackend.hpp:36
armnn::IBackendInternal::IBackendSpecificModelContextPtr
std::shared_ptr< IBackendModelContext > IBackendSpecificModelContextPtr
Definition: IBackendInternal.hpp:96
armnn::ClBackend::CreateMemoryManager
IBackendInternal::IMemoryManagerUniquePtr CreateMemoryManager() const override
Definition: ClBackend.cpp:50
armnn::ClBackend::ClBackend
ClBackend()
Definition: ClBackend.hpp:27