Compute Library
 21.02
CLCropResize.cpp
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2019-2021 Arm Limited.
3  *
4  * SPDX-License-Identifier: MIT
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a copy
7  * of this software and associated documentation files (the "Software"), to
8  * deal in the Software without restriction, including without limitation the
9  * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10  * sell copies of the Software, and to permit persons to whom the Software is
11  * furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in all
14  * copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22  * SOFTWARE.
23  */
25 
31 
32 #include <cstddef>
33 
34 namespace arm_compute
35 {
36 namespace
37 {
38 inline void configure_crop(const ICLTensor *input, ICLTensor *crop_boxes, ICLTensor *box_ind, ICLTensor *output, uint32_t crop_box_ind, Coordinates &start, Coordinates &end, uint32_t &batch_index)
39 {
40  batch_index = *(reinterpret_cast<int32_t *>(box_ind->ptr_to_element(Coordinates(crop_box_ind))));
41 
42  // _crop_box_ind is used to index crop_boxes and retrieve the appropriate crop box.
43  // The crop box is specified by normalized coordinates [y0, x0, y1, x1].
44  const float x0 = *reinterpret_cast<const float *>(crop_boxes->ptr_to_element(Coordinates(1, crop_box_ind)));
45  const float y0 = *reinterpret_cast<const float *>(crop_boxes->ptr_to_element(Coordinates(0, crop_box_ind)));
46  const float x1 = *reinterpret_cast<const float *>(crop_boxes->ptr_to_element(Coordinates(3, crop_box_ind)));
47  const float y1 = *reinterpret_cast<const float *>(crop_boxes->ptr_to_element(Coordinates(2, crop_box_ind)));
48  // The normalized coordinates are scaled to retrieve the floating point image coordinates which are rounded to integers.
49  start = Coordinates(std::floor(x0 * (input->info()->tensor_shape()[1] - 1) + 0.5f),
50  std::floor(y0 * (input->info()->tensor_shape()[2] - 1) + 0.5f));
51  end = Coordinates(std::floor(x1 * (input->info()->tensor_shape()[1] - 1) + 0.5f),
52  std::floor(y1 * (input->info()->tensor_shape()[2] - 1) + 0.5f));
53  const TensorShape out_shape(input->info()->tensor_shape()[0], static_cast<uint32_t>(abs(end[0] - start[0])) + 1, static_cast<uint32_t>(abs(end[1] - start[1])) + 1);
54  output->info()->set_tensor_shape(out_shape);
55 }
56 } // namespace
57 
59  : _input(nullptr), _boxes(nullptr), _box_ind(nullptr), _output(nullptr), _num_boxes(0), _method(), _extrapolation_value(0), _scale(), _copy(), _crop_results(), _scaled_results(), _internal_functions()
60 {
61 }
62 
63 CLCropResize::~CLCropResize() = default;
64 
65 Status CLCropResize::validate(const ITensorInfo *input, ITensorInfo *boxes, ITensorInfo *box_ind, const ITensorInfo *output,
66  Coordinates2D crop_size, InterpolationPolicy method, float extrapolation_value)
67 {
68  ARM_COMPUTE_RETURN_ERROR_ON(crop_size.x <= 0 || crop_size.y <= 0);
70  ARM_COMPUTE_RETURN_ERROR_ON(boxes->tensor_shape()[0] != 4);
71  ARM_COMPUTE_RETURN_ERROR_ON(boxes->tensor_shape()[1] != box_ind->tensor_shape()[0]);
72  TensorInfo temp_info;
73  ARM_COMPUTE_RETURN_ON_ERROR(CLCrop::validate(input->clone().get(), &temp_info, { 0, 0 }, { 1, 1 }, input->dimension(3) - 1, extrapolation_value));
74  if(output->total_size() > 0)
75  {
78  TensorShape out_shape(input->tensor_shape()[0], crop_size.x, crop_size.y, boxes->tensor_shape()[1]);
80  }
81  return Status{};
82 }
83 
84 void CLCropResize::configure(const ICLTensor *input, ICLTensor *boxes, ICLTensor *box_ind, ICLTensor *output, Coordinates2D crop_size,
85  InterpolationPolicy method, float extrapolation_value)
86 {
87  configure(CLKernelLibrary::get().get_compile_context(), input, boxes, box_ind, output, crop_size, method, extrapolation_value);
88 }
89 
90 void CLCropResize::configure(const CLCompileContext &compile_context, const ICLTensor *input, ICLTensor *boxes, ICLTensor *box_ind, ICLTensor *output, Coordinates2D crop_size,
91  InterpolationPolicy method, float extrapolation_value)
92 {
93  ARM_COMPUTE_ERROR_ON_NULLPTR(input, output, boxes, box_ind);
94  ARM_COMPUTE_ERROR_THROW_ON(CLCropResize::validate(input->info(), boxes->info(), box_ind->info(), output->info(), crop_size, method, extrapolation_value));
95 
96  TensorShape output_shape = TensorShape(input->info()->tensor_shape()[0], crop_size.x, crop_size.y, boxes->info()->tensor_shape()[1]);
98 
99  _num_boxes = boxes->info()->tensor_shape()[1];
100  TensorShape out_shape(input->info()->tensor_shape()[0], crop_size.x, crop_size.y);
101 
102  _input = input;
103  _boxes = boxes;
104  _box_ind = box_ind;
105  _output = output;
106  _method = method;
107  _extrapolation_value = extrapolation_value;
108 
109  // For each crop box:
110  // - The initial cropped image is produced as specified by boxes[i] from the 3D image input[box_ind[i]].
111  // Possibly using a CLCrop and up to four CLFills.
112  // - A tensor is required to hold this initial cropped image.
113  // - A scale function is used to resize the cropped image to the size specified by crop_size.
114  // - A tensor is required to hold the final scaled image before it is copied into the 4D output
115  // that will hold all final cropped and scaled 3D images using CLCopy.
116 
117  // The contents of _boxes and _box_ind are required to calculate the shape
118  // of the initial cropped image and thus are required to configure the
119  // kernels used for cropping and scaling.
120  _boxes->map(CLScheduler::get().queue());
121  _box_ind->map(CLScheduler::get().queue());
122  for(unsigned int num_box = 0; num_box < _num_boxes; ++num_box)
123  {
124  auto crop_tensor = std::make_unique<CLTensor>();
125  TensorInfo crop_result_info(1, DataType::F32);
126  crop_result_info.set_data_layout(DataLayout::NHWC);
127  crop_tensor->allocator()->init(crop_result_info);
128  _crop_results.emplace_back(std::move(crop_tensor));
129 
130  auto scale_tensor = std::make_unique<CLTensor>();
131  TensorInfo scaled_result_info(out_shape, 1, DataType::F32);
132  scaled_result_info.set_data_layout(DataLayout::NHWC);
133  scale_tensor->allocator()->init(scaled_result_info);
134  _scaled_results.emplace_back(std::move(scale_tensor));
135 
136  // Size of the crop box in _boxes has to be given before the configure
137  uint32_t batch_index;
138  Coordinates start{};
139  Coordinates end{};
140  configure_crop(_input, _boxes, _box_ind, _crop_results[num_box].get(), num_box, start, end, batch_index);
141 
142  auto scale_kernel = std::make_unique<CLScale>();
143  scale_kernel->configure(compile_context, _crop_results[num_box].get(), _scaled_results[num_box].get(), ScaleKernelInfo{ _method, BorderMode::CONSTANT, PixelValue(_extrapolation_value), SamplingPolicy::TOP_LEFT });
144  _scale.emplace_back(std::move(scale_kernel));
145 
147  win.set(3, Window::Dimension(num_box, num_box + 1, 1));
148 
149  auto copy_kernel = std::make_unique<CLCopy>();
150  copy_kernel->configure(compile_context, _scaled_results[num_box].get(), _output, &win);
151  _copy.emplace_back(std::move(copy_kernel));
152 
153  _crop_results[num_box]->allocator()->allocate();
154  _scaled_results[num_box]->allocator()->allocate();
155 
156  bool is_width_flipped = end[0] < start[0];
157  bool is_height_flipped = end[1] < start[1];
158  /** The number of rows out of bounds at the start and end of _crop_results[num_box].get(). */
159  std::array<int32_t, 2> rows_out_of_bounds{ 0 };
160  /** The number of columns out of bounds at the start and end of _crop_results[num_box].get(). */
161  std::array<int32_t, 2> cols_out_of_bounds{ 0 };
162  if(is_height_flipped)
163  {
164  rows_out_of_bounds[0] = start[1] >= static_cast<int32_t>(_input->info()->dimension(2)) ? std::min(start[1] - _input->info()->dimension(2) + 1, _crop_results[num_box].get()->info()->dimension(2)) : 0;
165  rows_out_of_bounds[1] = end[1] < 0 ? std::min(-end[1], static_cast<int32_t>(_crop_results[num_box].get()->info()->dimension(2))) : 0;
166  }
167  else
168  {
169  rows_out_of_bounds[0] = start[1] < 0 ? std::min(-start[1], static_cast<int32_t>(_crop_results[num_box].get()->info()->dimension(2))) : 0;
170  rows_out_of_bounds[1] = end[1] >= static_cast<int32_t>(_input->info()->dimension(2)) ? std::min(end[1] - _input->info()->dimension(2) + 1, _crop_results[num_box].get()->info()->dimension(2)) : 0;
171  }
172  if(is_width_flipped)
173  {
174  cols_out_of_bounds[0] = start[0] >= static_cast<int32_t>(_input->info()->dimension(1)) ? std::min(start[0] - _input->info()->dimension(1) + 1, _crop_results[num_box].get()->info()->dimension(1)) : 0;
175  cols_out_of_bounds[1] = end[0] < 0 ? std::min(-end[0], static_cast<int32_t>(_crop_results[num_box].get()->info()->dimension(1))) : 0;
176  }
177  else
178  {
179  cols_out_of_bounds[0] = start[0] < 0 ? std::min(-start[0], static_cast<int32_t>(_crop_results[num_box].get()->info()->dimension(1))) : 0;
180  cols_out_of_bounds[1] = end[0] >= static_cast<int32_t>(_input->info()->dimension(1)) ? std::min(end[0] - _input->info()->dimension(1) + 1, _crop_results[num_box].get()->info()->dimension(1)) : 0;
181  }
182 
183  Window full_window = calculate_max_window(*_crop_results[num_box].get()->info());
184 
185  // Full _crop_results[num_box].get() window:
186  // --------------------------------
187  // | Out of bounds |
188  // | rows before |
189  // |------------------------------|
190  // | Out of | In | Out of |
191  // | bounds | bounds | bounds |
192  // | cols | elements | cols |
193  // | before | copied | after |
194  // | | from input | |
195  // |------------------------------|
196  // | Out of bounds |
197  // | rows after |
198  // |------------------------------|
199  // Use a separate _crop_results[num_box].get() window for each section of the full _crop_results[num_box].get() window.
200  // Fill all _crop_results[num_box].get() rows that have no elements that are within the input bounds
201  // with the extrapolation value using memset.
202  // First for the rows before the in bounds rows.
203  if(rows_out_of_bounds[0] > 0)
204  {
205  Window slice_fill_rows_before(full_window);
206  slice_fill_rows_before.set(2, Window::Dimension(0, rows_out_of_bounds[0], 1));
207  auto kernel = std::make_unique<CLFill>();
208  kernel->configure(compile_context, _crop_results[num_box].get(), extrapolation_value, &slice_fill_rows_before);
209  //_internal_functions.emplace_back(std::move(kernel));
210  _internal_functions.push_back(std::move(kernel));
211  }
212 
213  Window slice_in(full_window);
214  slice_in.set(2, Window::Dimension(rows_out_of_bounds[0], _crop_results[num_box].get()->info()->dimension(2) - rows_out_of_bounds[1], 1));
215  slice_in.set(1, Window::Dimension(cols_out_of_bounds[0], _crop_results[num_box].get()->info()->dimension(1) - cols_out_of_bounds[1], 1));
216 
217  int rows_in_bounds = static_cast<int32_t>(_crop_results[num_box].get()->info()->dimension(2)) - rows_out_of_bounds[0] - rows_out_of_bounds[1];
218  if(rows_in_bounds > 0)
219  {
220  // Fill all elements that share a row with an in bounds element with the extrapolation value.
221  if(cols_out_of_bounds[0] > 0)
222  {
223  Window slice_fill_cols_before(slice_in);
224  slice_fill_cols_before.set(1, Window::Dimension(0, cols_out_of_bounds[0], 1));
225  auto kernel = std::make_unique<CLFill>();
226  kernel->configure(compile_context, _crop_results[num_box].get(), extrapolation_value, &slice_fill_cols_before);
227  //_internal_functions.emplace_back(std::move(kernel));
228  _internal_functions.push_back(std::move(kernel));
229  }
230 
231  if(cols_out_of_bounds[1] > 0)
232  {
233  Window slice_fill_cols_after(slice_in);
234  slice_fill_cols_after.set(1, Window::Dimension(_crop_results[num_box].get()->info()->dimension(1) - cols_out_of_bounds[1], _crop_results[num_box].get()->info()->dimension(1), 1));
235  auto kernel = std::make_unique<CLFill>();
236  kernel->configure(compile_context, _crop_results[num_box].get(), extrapolation_value, &slice_fill_cols_after);
237  //_internal_functions.emplace_back(std::move(kernel));
238  _internal_functions.push_back(std::move(kernel));
239  }
240 
241  // Copy all elements within the input bounds from the input tensor.
242  int cols_in_bounds = static_cast<int32_t>(_crop_results[num_box].get()->info()->dimension(1)) - cols_out_of_bounds[0] - cols_out_of_bounds[1];
243  if(cols_in_bounds > 0)
244  {
245  Coordinates2D start_in{ is_width_flipped ? start[0] - cols_out_of_bounds[0] : start[0] + cols_out_of_bounds[0],
246  is_height_flipped ? start[1] - rows_out_of_bounds[0] : start[1] + rows_out_of_bounds[0] };
247  Coordinates2D end_in{ is_width_flipped ? start_in.x - cols_in_bounds + 1 : start_in.x + cols_in_bounds - 1,
248  is_height_flipped ? start_in.y - rows_in_bounds + 1 : start_in.y + rows_in_bounds - 1 };
249  auto kernel = std::make_unique<CLCrop>();
250 
251  kernel->configure(compile_context, _input, _crop_results[num_box].get(), start_in, end_in, batch_index, extrapolation_value, &slice_in);
252  //_internal_functions.emplace_back(std::move(kernel));
253  _internal_functions.push_back(std::move(kernel));
254  }
255  }
256 
257  // Fill all rows after the in bounds elements with the extrapolation value.
258  if(rows_out_of_bounds[1] > 0)
259  {
260  Window slice_fill_rows_after(full_window);
261  slice_fill_rows_after.set(2, Window::Dimension(_crop_results[num_box].get()->info()->dimension(2) - rows_out_of_bounds[1], _crop_results[num_box].get()->info()->dimension(2), 1));
262  auto kernel = std::make_unique<CLFill>();
263  kernel->configure(compile_context, _crop_results[num_box].get(), extrapolation_value, &slice_fill_rows_after);
264  //_internal_functions.emplace_back(std::move(kernel));
265  _internal_functions.push_back(std::move(kernel));
266  }
267  }
268  _boxes->unmap(CLScheduler::get().queue());
269  _box_ind->unmap(CLScheduler::get().queue());
271 }
272 
274 {
275  ARM_COMPUTE_ERROR_ON_MSG(_output == nullptr, "Unconfigured function");
276 
277  for(unsigned int i = 0; i < _internal_functions.size(); ++i)
278  {
279  _internal_functions[i]->run();
280  }
281 
283  for(auto &kernel : _scale)
284  {
285  kernel->run();
286  }
288  for(auto &kernel : _copy)
289  {
290  kernel->run();
291  }
293 }
294 } // namespace arm_compute
Class describing the value of a pixel for any image format.
Definition: PixelValue.h:34
Window calculate_max_window(const ValidRegion &valid_region, const Steps &steps, bool skip_border, BorderSize border_size)
InterpolationPolicy
Interpolation method.
Definition: Types.h:392
std::vector< std::unique_ptr< CLTensor > > _crop_results
Definition: CLCropResize.h:131
CLCropResize()
Default constructor.
void map(cl::CommandQueue &q, bool blocking=true)
Enqueue a map operation of the allocated buffer on the given queue.
Definition: ICLTensor.cpp:35
Shape of a tensor.
Definition: TensorShape.h:39
~CLCropResize()
Default destructor.
#define ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_LAYOUT(...)
Definition: Validate.h:494
std::vector< std::unique_ptr< CLCopy > > _copy
Definition: CLCropResize.h:130
virtual size_t dimension(size_t index) const =0
Return the size of the requested dimension.
std::vector< std::unique_ptr< IFunction > > _internal_functions
Definition: CLCropResize.h:134
static CLScheduler & get()
Access the scheduler singleton.
#define ARM_COMPUTE_RETURN_ON_ERROR(status)
Checks if a status contains an error and returns it.
Definition: Error.h:204
1 channel, 1 F32 per channel
static CLKernelLibrary & get()
Access the KernelLibrary singleton.
Store the tensor&#39;s metadata.
Definition: ITensorInfo.h:40
#define ARM_COMPUTE_ERROR_THROW_ON(status)
Definition: Error.h:455
Describe one of the image&#39;s dimensions with a start, end and step.
Definition: Window.h:77
Status class.
Definition: Error.h:52
#define ARM_COMPUTE_RETURN_ERROR_ON(cond)
If the condition is true, an error is returned.
Definition: Error.h:296
#define ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DIMENSIONS(...)
Definition: Validate.h:288
Copyright (c) 2017-2021 Arm Limited.
int32_t x
X coordinates.
Definition: Types.h:465
std::vector< std::unique_ptr< CLTensor > > _scaled_results
Definition: CLCropResize.h:132
InterpolationPolicy _method
Definition: CLCropResize.h:126
Interface to enqueue OpenCL kernels and get/set the OpenCL CommandQueue and ICLTuner.
int32_t y
Y coordinates.
Definition: Types.h:466
virtual const TensorShape & tensor_shape() const =0
Size for each dimension of the tensor.
void unmap(cl::CommandQueue &q)
Enqueue an unmap operation of the allocated and mapped buffer on the given queue. ...
Definition: ICLTensor.cpp:40
#define ARM_COMPUTE_ERROR_ON_MSG(cond, msg)
Definition: Error.h:456
Coordinates of an item.
Definition: Coordinates.h:37
static Status validate(const ITensorInfo *input, ITensorInfo *boxes, ITensorInfo *box_ind, const ITensorInfo *output, Coordinates2D crop_size, InterpolationPolicy method, float extrapolation_value)
Static function to check if given info will lead to a valid configuration of NESlice.
bool auto_init_if_empty(ITensorInfo &info, const TensorShape &shape, int num_channels, DataType data_type, QuantizationInfo quantization_info=QuantizationInfo())
Auto initialize the tensor info (shape, number of channels and data type) if the current assignment i...
virtual std::unique_ptr< T > clone() const =0
Provide a clone of the current object of class T.
void run() override
Run the kernels contained in the function.
virtual ITensorInfo * info() const =0
Interface to be implemented by the child class to return the tensor&#39;s metadata.
Samples are taken at pixel top left corner.
void end(TokenStream &in, bool &valid)
Definition: MLGOParser.cpp:290
const ICLTensor * _input
Definition: CLCropResize.h:121
void set(size_t dimension, const Dimension &dim)
Set the values of a given dimension.
Definition: Window.inl:49
Output values are determined by averaging the source pixels whose areas fall under the area of the de...
CLCompileContext class.
void sync()
Blocks until all commands in the associated command queue have finished.
Definition: CLScheduler.cpp:72
static Status validate(const ITensorInfo *input, const ITensorInfo *output, Coordinates2D start, Coordinates2D end, uint32_t batch_index, float extrapolation_value=0, Window *output_window=nullptr)
Static function to check if given info will lead to a valid configuration of CLStridedSliceKernel.
Definition: CLCrop.cpp:70
ScaleKernelInfo info(interpolation_policy, default_border_mode, PixelValue(), sampling_policy, false)
Interface for OpenCL tensor.
Definition: ICLTensor.h:42
virtual size_t total_size() const =0
Returns the total size of the tensor in bytes.
Coordinate type.
Definition: Types.h:463
Num samples, height, width, channels.
void configure(const ICLTensor *input, ICLTensor *boxes, ICLTensor *box_ind, ICLTensor *output, Coordinates2D crop_size, InterpolationPolicy method=InterpolationPolicy::BILINEAR, float extrapolation_value=0)
Configure kernel.
std::vector< std::unique_ptr< CLScale > > _scale
Definition: CLCropResize.h:129
#define ARM_COMPUTE_ERROR_ON_NULLPTR(...)
Definition: Validate.h:161
Store the tensor&#39;s metadata.
Definition: TensorInfo.h:45
#define ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_NOT_IN(t,...)
Definition: Validate.h:694
Describe a multidimensional execution window.
Definition: Window.h:39