Compute Library
 21.11
CLCropResize.cpp
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2019-2021 Arm Limited.
3  *
4  * SPDX-License-Identifier: MIT
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a copy
7  * of this software and associated documentation files (the "Software"), to
8  * deal in the Software without restriction, including without limitation the
9  * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10  * sell copies of the Software, and to permit persons to whom the Software is
11  * furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in all
14  * copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22  * SOFTWARE.
23  */
25 
31 
32 #include "src/common/utils/Log.h"
33 
34 #include <cstddef>
35 
36 namespace arm_compute
37 {
38 namespace
39 {
40 inline void configure_crop(const ICLTensor *input, ICLTensor *crop_boxes, ICLTensor *box_ind, ICLTensor *output, uint32_t crop_box_ind, Coordinates &start, Coordinates &end, uint32_t &batch_index)
41 {
42  batch_index = *(reinterpret_cast<int32_t *>(box_ind->ptr_to_element(Coordinates(crop_box_ind))));
43 
44  // _crop_box_ind is used to index crop_boxes and retrieve the appropriate crop box.
45  // The crop box is specified by normalized coordinates [y0, x0, y1, x1].
46  const float x0 = *reinterpret_cast<const float *>(crop_boxes->ptr_to_element(Coordinates(1, crop_box_ind)));
47  const float y0 = *reinterpret_cast<const float *>(crop_boxes->ptr_to_element(Coordinates(0, crop_box_ind)));
48  const float x1 = *reinterpret_cast<const float *>(crop_boxes->ptr_to_element(Coordinates(3, crop_box_ind)));
49  const float y1 = *reinterpret_cast<const float *>(crop_boxes->ptr_to_element(Coordinates(2, crop_box_ind)));
50  // The normalized coordinates are scaled to retrieve the floating point image coordinates which are rounded to integers.
51  start = Coordinates(std::floor(x0 * (input->info()->tensor_shape()[1] - 1) + 0.5f),
52  std::floor(y0 * (input->info()->tensor_shape()[2] - 1) + 0.5f));
53  end = Coordinates(std::floor(x1 * (input->info()->tensor_shape()[1] - 1) + 0.5f),
54  std::floor(y1 * (input->info()->tensor_shape()[2] - 1) + 0.5f));
55  const TensorShape out_shape(input->info()->tensor_shape()[0], static_cast<uint32_t>(abs(end[0] - start[0])) + 1, static_cast<uint32_t>(abs(end[1] - start[1])) + 1);
56  output->info()->set_tensor_shape(out_shape);
57 }
58 } // namespace
59 
61  : _input(nullptr), _boxes(nullptr), _box_ind(nullptr), _output(nullptr), _num_boxes(0), _method(), _extrapolation_value(0), _scale(), _copy(), _crop_results(), _scaled_results(), _internal_functions()
62 {
63 }
64 
65 CLCropResize::~CLCropResize() = default;
66 
67 Status CLCropResize::validate(const ITensorInfo *input, ITensorInfo *boxes, ITensorInfo *box_ind, const ITensorInfo *output,
68  Coordinates2D crop_size, InterpolationPolicy method, float extrapolation_value)
69 {
70  ARM_COMPUTE_RETURN_ERROR_ON(crop_size.x <= 0 || crop_size.y <= 0);
72  ARM_COMPUTE_RETURN_ERROR_ON(boxes->tensor_shape()[0] != 4);
73  ARM_COMPUTE_RETURN_ERROR_ON(boxes->tensor_shape()[1] != box_ind->tensor_shape()[0]);
74  TensorInfo temp_info;
75  ARM_COMPUTE_RETURN_ON_ERROR(CLCrop::validate(input->clone().get(), &temp_info, { 0, 0 }, { 1, 1 }, input->dimension(3) - 1, extrapolation_value));
76  if(output->total_size() > 0)
77  {
80  TensorShape out_shape(input->tensor_shape()[0], crop_size.x, crop_size.y, boxes->tensor_shape()[1]);
82  }
83  return Status{};
84 }
85 
86 void CLCropResize::configure(const ICLTensor *input, ICLTensor *boxes, ICLTensor *box_ind, ICLTensor *output, Coordinates2D crop_size,
87  InterpolationPolicy method, float extrapolation_value)
88 {
89  configure(CLKernelLibrary::get().get_compile_context(), input, boxes, box_ind, output, crop_size, method, extrapolation_value);
90 }
91 
92 void CLCropResize::configure(const CLCompileContext &compile_context, const ICLTensor *input, ICLTensor *boxes, ICLTensor *box_ind, ICLTensor *output, Coordinates2D crop_size,
93  InterpolationPolicy method, float extrapolation_value)
94 {
95  ARM_COMPUTE_ERROR_ON_NULLPTR(input, output, boxes, box_ind);
96  ARM_COMPUTE_ERROR_THROW_ON(CLCropResize::validate(input->info(), boxes->info(), box_ind->info(), output->info(), crop_size, method, extrapolation_value));
97  ARM_COMPUTE_LOG_PARAMS(input, boxes, box_ind, output, crop_size, method, extrapolation_value);
98 
99  TensorShape output_shape = TensorShape(input->info()->tensor_shape()[0], crop_size.x, crop_size.y, boxes->info()->tensor_shape()[1]);
101 
102  _num_boxes = boxes->info()->tensor_shape()[1];
103  TensorShape out_shape(input->info()->tensor_shape()[0], crop_size.x, crop_size.y);
104 
105  _input = input;
106  _boxes = boxes;
107  _box_ind = box_ind;
108  _output = output;
109  _method = method;
110  _extrapolation_value = extrapolation_value;
111 
112  // For each crop box:
113  // - The initial cropped image is produced as specified by boxes[i] from the 3D image input[box_ind[i]].
114  // Possibly using a CLCrop and up to four CLFills.
115  // - A tensor is required to hold this initial cropped image.
116  // - A scale function is used to resize the cropped image to the size specified by crop_size.
117  // - A tensor is required to hold the final scaled image before it is copied into the 4D output
118  // that will hold all final cropped and scaled 3D images using CLCopy.
119 
120  // The contents of _boxes and _box_ind are required to calculate the shape
121  // of the initial cropped image and thus are required to configure the
122  // kernels used for cropping and scaling.
123  _boxes->map(CLScheduler::get().queue());
124  _box_ind->map(CLScheduler::get().queue());
125  for(unsigned int num_box = 0; num_box < _num_boxes; ++num_box)
126  {
127  auto crop_tensor = std::make_unique<CLTensor>();
128  TensorInfo crop_result_info(1, DataType::F32);
129  crop_result_info.set_data_layout(DataLayout::NHWC);
130  crop_tensor->allocator()->init(crop_result_info);
131  _crop_results.emplace_back(std::move(crop_tensor));
132 
133  auto scale_tensor = std::make_unique<CLTensor>();
134  TensorInfo scaled_result_info(out_shape, 1, DataType::F32);
135  scaled_result_info.set_data_layout(DataLayout::NHWC);
136  scale_tensor->allocator()->init(scaled_result_info);
137  _scaled_results.emplace_back(std::move(scale_tensor));
138 
139  // Size of the crop box in _boxes has to be given before the configure
140  uint32_t batch_index;
141  Coordinates start{};
142  Coordinates end{};
143  configure_crop(_input, _boxes, _box_ind, _crop_results[num_box].get(), num_box, start, end, batch_index);
144 
145  auto scale_kernel = std::make_unique<CLScale>();
146  scale_kernel->configure(compile_context, _crop_results[num_box].get(), _scaled_results[num_box].get(), ScaleKernelInfo{ _method, BorderMode::CONSTANT, PixelValue(_extrapolation_value), SamplingPolicy::TOP_LEFT });
147  _scale.emplace_back(std::move(scale_kernel));
148 
150  win.set(3, Window::Dimension(num_box, num_box + 1, 1));
151 
152  auto copy_kernel = std::make_unique<CLCopy>();
153  copy_kernel->configure(compile_context, _scaled_results[num_box].get(), _output, &win);
154  _copy.emplace_back(std::move(copy_kernel));
155 
156  _crop_results[num_box]->allocator()->allocate();
157  _scaled_results[num_box]->allocator()->allocate();
158 
159  bool is_width_flipped = end[0] < start[0];
160  bool is_height_flipped = end[1] < start[1];
161  /** The number of rows out of bounds at the start and end of _crop_results[num_box].get(). */
162  std::array<int32_t, 2> rows_out_of_bounds{ 0 };
163  /** The number of columns out of bounds at the start and end of _crop_results[num_box].get(). */
164  std::array<int32_t, 2> cols_out_of_bounds{ 0 };
165  if(is_height_flipped)
166  {
167  rows_out_of_bounds[0] = start[1] >= static_cast<int32_t>(_input->info()->dimension(2)) ? std::min(start[1] - _input->info()->dimension(2) + 1, _crop_results[num_box].get()->info()->dimension(2)) : 0;
168  rows_out_of_bounds[1] = end[1] < 0 ? std::min(-end[1], static_cast<int32_t>(_crop_results[num_box].get()->info()->dimension(2))) : 0;
169  }
170  else
171  {
172  rows_out_of_bounds[0] = start[1] < 0 ? std::min(-start[1], static_cast<int32_t>(_crop_results[num_box].get()->info()->dimension(2))) : 0;
173  rows_out_of_bounds[1] = end[1] >= static_cast<int32_t>(_input->info()->dimension(2)) ? std::min(end[1] - _input->info()->dimension(2) + 1, _crop_results[num_box].get()->info()->dimension(2)) : 0;
174  }
175  if(is_width_flipped)
176  {
177  cols_out_of_bounds[0] = start[0] >= static_cast<int32_t>(_input->info()->dimension(1)) ? std::min(start[0] - _input->info()->dimension(1) + 1, _crop_results[num_box].get()->info()->dimension(1)) : 0;
178  cols_out_of_bounds[1] = end[0] < 0 ? std::min(-end[0], static_cast<int32_t>(_crop_results[num_box].get()->info()->dimension(1))) : 0;
179  }
180  else
181  {
182  cols_out_of_bounds[0] = start[0] < 0 ? std::min(-start[0], static_cast<int32_t>(_crop_results[num_box].get()->info()->dimension(1))) : 0;
183  cols_out_of_bounds[1] = end[0] >= static_cast<int32_t>(_input->info()->dimension(1)) ? std::min(end[0] - _input->info()->dimension(1) + 1, _crop_results[num_box].get()->info()->dimension(1)) : 0;
184  }
185 
186  Window full_window = calculate_max_window(*_crop_results[num_box].get()->info());
187 
188  // Full _crop_results[num_box].get() window:
189  // --------------------------------
190  // | Out of bounds |
191  // | rows before |
192  // |------------------------------|
193  // | Out of | In | Out of |
194  // | bounds | bounds | bounds |
195  // | cols | elements | cols |
196  // | before | copied | after |
197  // | | from input | |
198  // |------------------------------|
199  // | Out of bounds |
200  // | rows after |
201  // |------------------------------|
202  // Use a separate _crop_results[num_box].get() window for each section of the full _crop_results[num_box].get() window.
203  // Fill all _crop_results[num_box].get() rows that have no elements that are within the input bounds
204  // with the extrapolation value using memset.
205  // First for the rows before the in bounds rows.
206  if(rows_out_of_bounds[0] > 0)
207  {
208  Window slice_fill_rows_before(full_window);
209  slice_fill_rows_before.set(2, Window::Dimension(0, rows_out_of_bounds[0], 1));
210  auto kernel = std::make_unique<CLFill>();
211  kernel->configure(compile_context, _crop_results[num_box].get(), extrapolation_value, &slice_fill_rows_before);
212  //_internal_functions.emplace_back(std::move(kernel));
213  _internal_functions.push_back(std::move(kernel));
214  }
215 
216  Window slice_in(full_window);
217  slice_in.set(2, Window::Dimension(rows_out_of_bounds[0], _crop_results[num_box].get()->info()->dimension(2) - rows_out_of_bounds[1], 1));
218  slice_in.set(1, Window::Dimension(cols_out_of_bounds[0], _crop_results[num_box].get()->info()->dimension(1) - cols_out_of_bounds[1], 1));
219 
220  int rows_in_bounds = static_cast<int32_t>(_crop_results[num_box].get()->info()->dimension(2)) - rows_out_of_bounds[0] - rows_out_of_bounds[1];
221  if(rows_in_bounds > 0)
222  {
223  // Fill all elements that share a row with an in bounds element with the extrapolation value.
224  if(cols_out_of_bounds[0] > 0)
225  {
226  Window slice_fill_cols_before(slice_in);
227  slice_fill_cols_before.set(1, Window::Dimension(0, cols_out_of_bounds[0], 1));
228  auto kernel = std::make_unique<CLFill>();
229  kernel->configure(compile_context, _crop_results[num_box].get(), extrapolation_value, &slice_fill_cols_before);
230  //_internal_functions.emplace_back(std::move(kernel));
231  _internal_functions.push_back(std::move(kernel));
232  }
233 
234  if(cols_out_of_bounds[1] > 0)
235  {
236  Window slice_fill_cols_after(slice_in);
237  slice_fill_cols_after.set(1, Window::Dimension(_crop_results[num_box].get()->info()->dimension(1) - cols_out_of_bounds[1], _crop_results[num_box].get()->info()->dimension(1), 1));
238  auto kernel = std::make_unique<CLFill>();
239  kernel->configure(compile_context, _crop_results[num_box].get(), extrapolation_value, &slice_fill_cols_after);
240  //_internal_functions.emplace_back(std::move(kernel));
241  _internal_functions.push_back(std::move(kernel));
242  }
243 
244  // Copy all elements within the input bounds from the input tensor.
245  int cols_in_bounds = static_cast<int32_t>(_crop_results[num_box].get()->info()->dimension(1)) - cols_out_of_bounds[0] - cols_out_of_bounds[1];
246  if(cols_in_bounds > 0)
247  {
248  Coordinates2D start_in{ is_width_flipped ? start[0] - cols_out_of_bounds[0] : start[0] + cols_out_of_bounds[0],
249  is_height_flipped ? start[1] - rows_out_of_bounds[0] : start[1] + rows_out_of_bounds[0] };
250  Coordinates2D end_in{ is_width_flipped ? start_in.x - cols_in_bounds + 1 : start_in.x + cols_in_bounds - 1,
251  is_height_flipped ? start_in.y - rows_in_bounds + 1 : start_in.y + rows_in_bounds - 1 };
252  auto kernel = std::make_unique<CLCrop>();
253 
254  kernel->configure(compile_context, _input, _crop_results[num_box].get(), start_in, end_in, batch_index, extrapolation_value, &slice_in);
255  //_internal_functions.emplace_back(std::move(kernel));
256  _internal_functions.push_back(std::move(kernel));
257  }
258  }
259 
260  // Fill all rows after the in bounds elements with the extrapolation value.
261  if(rows_out_of_bounds[1] > 0)
262  {
263  Window slice_fill_rows_after(full_window);
264  slice_fill_rows_after.set(2, Window::Dimension(_crop_results[num_box].get()->info()->dimension(2) - rows_out_of_bounds[1], _crop_results[num_box].get()->info()->dimension(2), 1));
265  auto kernel = std::make_unique<CLFill>();
266  kernel->configure(compile_context, _crop_results[num_box].get(), extrapolation_value, &slice_fill_rows_after);
267  //_internal_functions.emplace_back(std::move(kernel));
268  _internal_functions.push_back(std::move(kernel));
269  }
270  }
271  _boxes->unmap(CLScheduler::get().queue());
272  _box_ind->unmap(CLScheduler::get().queue());
274 }
275 
277 {
278  ARM_COMPUTE_ERROR_ON_MSG(_output == nullptr, "Unconfigured function");
279 
280  for(unsigned int i = 0; i < _internal_functions.size(); ++i)
281  {
282  _internal_functions[i]->run();
283  }
284 
286  for(auto &kernel : _scale)
287  {
288  kernel->run();
289  }
291  for(auto &kernel : _copy)
292  {
293  kernel->run();
294  }
296 }
297 } // namespace arm_compute
Class describing the value of a pixel for any image format.
Definition: PixelValue.h:34
Window calculate_max_window(const ValidRegion &valid_region, const Steps &steps, bool skip_border, BorderSize border_size)
InterpolationPolicy
Interpolation method.
Definition: Types.h:398
std::vector< std::unique_ptr< CLTensor > > _crop_results
Definition: CLCropResize.h:139
CLCropResize()
Default constructor.
void map(cl::CommandQueue &q, bool blocking=true)
Enqueue a map operation of the allocated buffer on the given queue.
Definition: ICLTensor.cpp:35
Shape of a tensor.
Definition: TensorShape.h:39
~CLCropResize()
Default destructor.
#define ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_LAYOUT(...)
Definition: Validate.h:490
std::vector< std::unique_ptr< CLCopy > > _copy
Definition: CLCropResize.h:138
virtual size_t dimension(size_t index) const =0
Return the size of the requested dimension.
std::vector< std::unique_ptr< IFunction > > _internal_functions
Definition: CLCropResize.h:142
static CLScheduler & get()
Access the scheduler singleton.
#define ARM_COMPUTE_RETURN_ON_ERROR(status)
Checks if a status contains an error and returns it.
Definition: Error.h:204
1 channel, 1 F32 per channel
static CLKernelLibrary & get()
Access the KernelLibrary singleton.
Store the tensor&#39;s metadata.
Definition: ITensorInfo.h:40
#define ARM_COMPUTE_ERROR_THROW_ON(status)
Definition: Error.h:455
Describe one of the image&#39;s dimensions with a start, end and step.
Definition: Window.h:77
Status class.
Definition: Error.h:52
#define ARM_COMPUTE_RETURN_ERROR_ON(cond)
If the condition is true, an error is returned.
Definition: Error.h:296
#define ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DIMENSIONS(...)
Definition: Validate.h:284
Copyright (c) 2017-2021 Arm Limited.
int32_t x
X coordinates.
Definition: Types.h:424
std::vector< std::unique_ptr< CLTensor > > _scaled_results
Definition: CLCropResize.h:140
InterpolationPolicy _method
Definition: CLCropResize.h:134
Interface to enqueue OpenCL kernels and get/set the OpenCL CommandQueue and ICLTuner.
int32_t y
Y coordinates.
Definition: Types.h:425
virtual const TensorShape & tensor_shape() const =0
Size for each dimension of the tensor.
void unmap(cl::CommandQueue &q)
Enqueue an unmap operation of the allocated and mapped buffer on the given queue. ...
Definition: ICLTensor.cpp:40
#define ARM_COMPUTE_ERROR_ON_MSG(cond, msg)
Definition: Error.h:456
Coordinates of an item.
Definition: Coordinates.h:37
static Status validate(const ITensorInfo *input, ITensorInfo *boxes, ITensorInfo *box_ind, const ITensorInfo *output, Coordinates2D crop_size, InterpolationPolicy method, float extrapolation_value)
Static function to check if given info will lead to a valid configuration of NESlice.
bool auto_init_if_empty(ITensorInfo &info, const TensorShape &shape, int num_channels, DataType data_type, QuantizationInfo quantization_info=QuantizationInfo())
Auto initialize the tensor info (shape, number of channels and data type) if the current assignment i...
virtual std::unique_ptr< T > clone() const =0
Provide a clone of the current object of class T.
void run() override
Run the kernels contained in the function.
virtual ITensorInfo * info() const =0
Interface to be implemented by the child class to return the tensor&#39;s metadata.
Samples are taken at pixel top left corner.
void end(TokenStream &in, bool &valid)
Definition: MLGOParser.cpp:290
const ICLTensor * _input
Definition: CLCropResize.h:129
void set(size_t dimension, const Dimension &dim)
Set the values of a given dimension.
Definition: Window.inl:49
Output values are determined by averaging the source pixels whose areas fall under the area of the de...
CLCompileContext class.
void sync()
Blocks until all commands in the associated command queue have finished.
Definition: CLScheduler.cpp:70
static Status validate(const ITensorInfo *input, const ITensorInfo *output, Coordinates2D start, Coordinates2D end, uint32_t batch_index, float extrapolation_value=0, Window *output_window=nullptr)
Static function to check if given info will lead to a valid configuration of CLStridedSliceKernel.
Definition: CLCrop.cpp:73
ScaleKernelInfo info(interpolation_policy, default_border_mode, PixelValue(), sampling_policy, false)
Interface for OpenCL tensor.
Definition: ICLTensor.h:42
virtual size_t total_size() const =0
Returns the total size of the tensor in bytes.
Coordinate type.
Definition: Types.h:422
Num samples, height, width, channels.
void configure(const ICLTensor *input, ICLTensor *boxes, ICLTensor *box_ind, ICLTensor *output, Coordinates2D crop_size, InterpolationPolicy method=InterpolationPolicy::BILINEAR, float extrapolation_value=0)
Configure kernel.
std::vector< std::unique_ptr< CLScale > > _scale
Definition: CLCropResize.h:137
#define ARM_COMPUTE_LOG_PARAMS(...)
#define ARM_COMPUTE_ERROR_ON_NULLPTR(...)
Definition: Validate.h:157
Store the tensor&#39;s metadata.
Definition: TensorInfo.h:43
#define ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_NOT_IN(t,...)
Definition: Validate.h:690
Describe a multidimensional execution window.
Definition: Window.h:39