Compute Library
 21.05
CLTensorAllocator.cpp
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2016-2021 Arm Limited.
3  *
4  * SPDX-License-Identifier: MIT
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a copy
7  * of this software and associated documentation files (the "Software"), to
8  * deal in the Software without restriction, including without limitation the
9  * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10  * sell copies of the Software, and to permit persons to whom the Software is
11  * furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in all
14  * copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22  * SOFTWARE.
23  */
25 
26 #include "arm_compute/core/Error.h"
30 
31 namespace arm_compute
32 {
33 const cl::Buffer CLTensorAllocator::_empty_buffer = cl::Buffer();
34 namespace
35 {
36 /** Global user-defined allocator that can be used for all internal allocations of a CLTensor */
37 static IAllocator *static_global_cl_allocator = nullptr;
38 
39 /** Helper function used to allocate the backing memory of a tensor
40  *
41  * @param[in] context OpenCL context to use
42  * @param[in] size Size of the allocation
43  * @param[in] alignment Alignment of the allocation
44  *
45  * @return A wrapped memory region
46  */
47 std::unique_ptr<ICLMemoryRegion> allocate_region(CLCoreRuntimeContext *ctx, size_t size, cl_uint alignment)
48 {
49  // Try fine-grain SVM
50  std::unique_ptr<ICLMemoryRegion> region = std::make_unique<CLFineSVMMemoryRegion>(ctx,
51  CL_MEM_READ_WRITE | CL_MEM_SVM_FINE_GRAIN_BUFFER,
52  size,
53  alignment);
54 
55  // Try coarse-grain SVM in case of failure
56  if(region != nullptr && region->ptr() == nullptr)
57  {
58  region = std::make_unique<CLCoarseSVMMemoryRegion>(ctx, CL_MEM_READ_WRITE, size, alignment);
59  }
60  // Try legacy buffer memory in case of failure
61  if(region != nullptr && region->ptr() == nullptr)
62  {
63  region = std::make_unique<CLBufferMemoryRegion>(ctx, CL_MEM_ALLOC_HOST_PTR | CL_MEM_READ_WRITE, size);
64  }
65  return region;
66 }
67 /** Clears quantization arrays
68  *
69  * @param[in, out] scale Quantization scale array
70  * @param[in, out] offset Quantization offset array
71  */
72 void clear_quantization_arrays(CLFloatArray &scale, CLInt32Array &offset)
73 {
74  // Clear arrays
75  scale = CLFloatArray();
76  offset = CLInt32Array();
77 }
78 /** Helper function used to create quantization data arrays
79  *
80  * @param[in, out] scale Quantization scale array
81  * @param[in, out] offset Quantization offset array
82  * @param[in] qinfo Quantization info
83  * @param[in] pad_size Pad size to use in case array needs to be padded for computation purposes
84  */
85 void populate_quantization_info(CLFloatArray &scale, CLInt32Array &offset, const QuantizationInfo &qinfo, size_t pad_size)
86 {
87  clear_quantization_arrays(scale, offset);
88 
89  // Create scale array
90  const std::vector<float> &qscale = qinfo.scale();
91  const size_t num_elements = qscale.size();
92  const size_t element_size = sizeof(std::remove_reference<decltype(qscale)>::type::value_type);
93  scale = CLFloatArray(num_elements + pad_size);
94  scale.resize(num_elements);
95  CLScheduler::get().queue().enqueueWriteBuffer(scale.cl_buffer(), CL_TRUE, 0, num_elements * element_size, qinfo.scale().data());
96 
97  if(!qinfo.offset().empty())
98  {
99  // Create offset array
100  const std::vector<int32_t> &qoffset = qinfo.offset();
101  const size_t offset_element_size = sizeof(std::remove_reference<decltype(qoffset)>::type::value_type);
102  offset = CLInt32Array(num_elements + pad_size);
103  offset.resize(num_elements);
104  CLScheduler::get().queue().enqueueWriteBuffer(offset.cl_buffer(), CL_TRUE, 0, num_elements * offset_element_size, qinfo.offset().data());
105  }
106 }
107 } // namespace
108 
110  : _ctx(ctx), _owner(owner), _associated_memory_group(nullptr), _memory(), _mapping(nullptr), _scale(), _offset()
111 {
112 }
113 
115 {
116  return { &_scale, &_offset };
117 }
118 
120 {
121  return _mapping;
122 }
123 
124 const cl::Buffer &CLTensorAllocator::cl_data() const
125 {
126  return _memory.region() == nullptr ? _empty_buffer : _memory.cl_region()->cl_data();
127 }
128 
130 {
131  // Allocate tensor backing memory
132  if(_associated_memory_group == nullptr)
133  {
134  // Perform memory allocation
135  if(static_global_cl_allocator != nullptr)
136  {
137  _memory.set_owned_region(static_global_cl_allocator->make_region(info().total_size(), 0));
138  }
139  else if(_ctx == nullptr)
140  {
141  auto legacy_ctx = CLCoreRuntimeContext(nullptr, CLScheduler::get().context(), CLScheduler::get().queue());
142  _memory.set_owned_region(allocate_region(&legacy_ctx, info().total_size(), 0));
143  }
144  else
145  {
146  _memory.set_owned_region(allocate_region(_ctx->core_runtime_context(), info().total_size(), 0));
147  }
148  }
149  else
150  {
151  // Finalize memory management instead
152  _associated_memory_group->finalize_memory(_owner, _memory, info().total_size(), alignment());
153  }
154 
155  // Allocate and fill the quantization parameter arrays
157  {
158  const size_t pad_size = 0;
159  populate_quantization_info(_scale, _offset, info().quantization_info(), pad_size);
160  }
161 
162  // Lock allocator
163  info().set_is_resizable(false);
164 }
165 
167 {
168  _mapping = nullptr;
169  _memory.set_region(nullptr);
170  clear_quantization_arrays(_scale, _offset);
171  info().set_is_resizable(true);
172 }
173 
175 {
176  ARM_COMPUTE_RETURN_ERROR_ON(buffer.get() == nullptr);
177  ARM_COMPUTE_RETURN_ERROR_ON(buffer.getInfo<CL_MEM_SIZE>() < info().total_size());
178  ARM_COMPUTE_RETURN_ERROR_ON(buffer.getInfo<CL_MEM_CONTEXT>().get() != CLScheduler::get().context().get());
179  ARM_COMPUTE_RETURN_ERROR_ON(_associated_memory_group != nullptr);
180 
181  if(_ctx == nullptr)
182  {
183  auto legacy_ctx = CLCoreRuntimeContext(nullptr, CLScheduler::get().context(), CLScheduler::get().queue());
184  _memory.set_owned_region(std::make_unique<CLBufferMemoryRegion>(buffer, &legacy_ctx));
185  }
186  else
187  {
188  _memory.set_owned_region(std::make_unique<CLBufferMemoryRegion>(buffer, _ctx->core_runtime_context()));
189  }
190 
191  info().set_is_resizable(false);
192  return Status{};
193 }
194 
196 {
197  ARM_COMPUTE_ERROR_ON(associated_memory_group == nullptr);
198  ARM_COMPUTE_ERROR_ON(_associated_memory_group != nullptr && _associated_memory_group != associated_memory_group);
199  ARM_COMPUTE_ERROR_ON(_memory.region() != nullptr && _memory.cl_region()->cl_data().get() != nullptr);
200 
201  _associated_memory_group = associated_memory_group;
202 }
203 
205 {
206  static_global_cl_allocator = allocator;
207 }
208 
209 uint8_t *CLTensorAllocator::lock()
210 {
211  if(_ctx)
212  {
213  return map(_ctx->gpu_scheduler()->queue(), true);
214  }
215  else
216  {
217  return map(CLScheduler::get().queue(), true);
218  }
219 }
220 
221 void CLTensorAllocator::unlock()
222 {
223  ARM_COMPUTE_ERROR_ON(_memory.region() == nullptr);
224  if(_ctx)
225  {
226  unmap(_ctx->gpu_scheduler()->queue(), reinterpret_cast<uint8_t *>(_memory.region()->buffer()));
227  }
228  else
229  {
230  //Legacy singleton api
231  unmap(CLScheduler::get().queue(), reinterpret_cast<uint8_t *>(_memory.region()->buffer()));
232  }
233 }
234 
235 uint8_t *CLTensorAllocator::map(cl::CommandQueue &q, bool blocking)
236 {
237  ARM_COMPUTE_ERROR_ON(_mapping != nullptr);
238  ARM_COMPUTE_ERROR_ON(_memory.region() == nullptr);
239  ARM_COMPUTE_ERROR_ON(_memory.region()->buffer() != nullptr);
240 
241  _mapping = reinterpret_cast<uint8_t *>(_memory.cl_region()->map(q, blocking));
242  return _mapping;
243 }
244 
245 void CLTensorAllocator::unmap(cl::CommandQueue &q, uint8_t *mapping)
246 {
247  ARM_COMPUTE_ERROR_ON(_mapping == nullptr);
248  ARM_COMPUTE_ERROR_ON(_mapping != mapping);
249  ARM_COMPUTE_ERROR_ON(_memory.region() == nullptr);
250  ARM_COMPUTE_ERROR_ON(_memory.region()->buffer() == nullptr);
251  ARM_COMPUTE_UNUSED(mapping);
252 
253  _memory.cl_region()->unmap(q);
254  _mapping = nullptr;
255 }
256 } // namespace arm_compute
Memory group interface.
Definition: IMemoryGroup.h:37
__global uchar * offset(const Image *img, int x, int y)
Get the pointer position of a Image.
Definition: helpers.h:861
const std::vector< int32_t > & offset() const
Offset vector accessor.
static void set_global_allocator(IAllocator *allocator)
Sets global allocator that will be used by all CLTensor objects.
static CLScheduler & get()
Access the scheduler singleton.
CLQuantization quantization() const
Wrapped quantization info data accessor.
#define ARM_COMPUTE_ERROR_ON(cond)
If the condition is true then an error message is printed and an exception thrown.
Definition: Error.h:466
virtual void * buffer()=0
Returns the pointer to the allocated data.
Status class.
Definition: Error.h:52
CLCoreRuntimeContext * core_runtime_context()
#define ARM_COMPUTE_RETURN_ERROR_ON(cond)
If the condition is true, an error is returned.
Definition: Error.h:296
OpenCL quantization data.
Definition: CLTypes.h:62
Copyright (c) 2017-2021 Arm Limited.
Status import_memory(cl::Buffer buffer)
Import an existing memory as a tensor's backing memory.
const cl::Buffer & cl_data() const
Returns the underlying CL buffer.
void set_associated_memory_group(IMemoryGroup *associated_memory_group)
Associates the tensor with a memory group.
const DataType data_type
Definition: Im2Col.cpp:150
Interface to enqueue OpenCL kernels and get/set the OpenCL CommandQueue and ICLTuner.
CLArray< cl_float > CLFloatArray
OpenCL Array of floats.
Definition: CLArray.h:116
uint8_t * data()
Interface to be implemented by the child class to return the pointer to the mapped data.
#define ARM_COMPUTE_UNUSED(...)
To avoid unused variables warnings.
Definition: Error.h:152
uint8_t * map(cl::CommandQueue &q, bool blocking)
Enqueue a map operation of the allocated buffer on the given queue.
cl::Context & context()
Accessor for the associated CL context.
Definition: CLScheduler.cpp:32
virtual void finalize_memory(IMemoryManageable *obj, IMemory &obj_memory, size_t size, size_t alignment)=0
Finalizes memory for a given object.
Core runtime context for OpenCL.
bool is_data_type_quantized_per_channel(DataType dt)
Check if a given data type is of per channel type.
Definition: Utils.h:1044
void set_owned_region(std::unique_ptr< IMemoryRegion > region) final
Sets a memory region.
Definition: CLMemory.cpp:76
size_t total_size() const override
Returns the total size of the tensor in bytes.
Definition: TensorInfo.h:250
CLTensorAllocator(IMemoryManageable *owner=nullptr, CLRuntimeContext *ctx=nullptr)
Default constructor.
IMemoryRegion * region() final
Region accessor.
Definition: CLMemory.cpp:59
size_t alignment() const
Return underlying's tensor buffer alignment.
const std::vector< float > & scale() const
Scale vector accessor.
ICLMemoryRegion * cl_region()
OpenCL Region accessor.
Definition: CLMemory.cpp:49
void unmap(cl::CommandQueue &q, uint8_t *mapping)
Enqueue an unmap operation of the allocated buffer on the given queue.
ITensorInfo & set_is_resizable(bool is_resizable) override
Set the flag whether the tensor size can be changed.
Definition: TensorInfo.h:270
cl::CommandQueue & queue()
Accessor for the associated CL command queue.
Definition: CLScheduler.cpp:39
input allocator() -> allocate()
virtual void * map(cl::CommandQueue &q, bool blocking)=0
Enqueue a map operation of the allocated buffer on the given queue.
TensorInfo & info()
Return a reference to the tensor's metadata.
void set_region(IMemoryRegion *region) final
Sets a memory region.
Definition: CLMemory.cpp:69
void allocate() override
Allocate size specified by TensorInfo of OpenCL memory.
CLArray< cl_int > CLInt32Array
OpenCL Array of int32s.
Definition: CLArray.h:114
Interface of an object than can be memory managed.
Definition: IMemoryGroup.h:69
const QuantizationInfo qinfo
Definition: Im2Col.cpp:155
const cl::Buffer & cl_data() const
Interface to be implemented by the child class to return the pointer to the CL data.
void free() override
Free allocated OpenCL memory.
Allocator interface.
Definition: IAllocator.h:35
virtual void unmap(cl::CommandQueue &q)=0
Enqueue an unmap operation of the allocated buffer on the given queue.