Compute Library
 21.11
fp32.cpp
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2021 Arm Limited.
3  *
4  * SPDX-License-Identifier: MIT
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a copy
7  * of this software and associated documentation files (the "Software"), to
8  * deal in the Software without restriction, including without limitation the
9  * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10  * sell copies of the Software, and to permit persons to whom the Software is
11  * furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in all
14  * copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22  * SOFTWARE.
23  */
26 #include "arm_compute/core/Types.h"
31 
32 namespace arm_compute
33 {
34 namespace cpu
35 {
36 namespace
37 {
38 void pooling2_f32_maxpool_indices(const ITensor *src, ITensor *dst0, ITensor *dst1, PoolingLayerInfo &pool_info, const Window &window_src, const Window &window)
39 {
40  const int window_start_x = window.x().start();
41  const int window_end_x = window.x().end();
42  const int window_step_x = 4;
43 
44  Window window_out = window;
45  window_out.set(Window::DimX, Window::Dimension(0, 1, 1));
46 
47  Iterator in(src, window_src);
48  Iterator out(dst0, window_out);
49  Iterator indices(dst1, window_out);
50 
51  const int pool_pad_top = pool_info.pad_stride_info.pad_top();
52  const int pool_pad_left = pool_info.pad_stride_info.pad_left();
53 
54  int pool_stride_x = 0;
55  int pool_stride_y = 0;
56  std::tie(pool_stride_x, pool_stride_y) = pool_info.pad_stride_info.stride();
57 
58  float32x4_t vres;
59  float res;
60 
61  const int pad_right = src->info()->padding().right;
62  const int pad_left = src->info()->padding().left;
63  const int pad_horizontal = pad_right + pad_left;
64  const int in_stride_y = static_cast<int>(src->info()->strides_in_bytes().y());
65  const int in_stride_z = static_cast<int>(src->info()->strides_in_bytes().z());
66 
67  execute_window_loop(window_out, [&](const Coordinates & id)
68  {
69  const int idx_width = id.y() * pool_stride_x;
70  const int idx_height = id.z() * pool_stride_y;
71  const int pool_limit_y = pool_pad_top - idx_height;
72  const int pool_limit_x = pool_pad_left - idx_width;
73 
74  const int pool_start_y = std::max(0, window_src.z().start() + pool_limit_y);
75  const int pool_start_x = std::max(0, window_src.y().start() + pool_limit_x);
76 
77  const int in_x0_offset = (pool_start_x - pool_pad_left) * static_cast<int>(src->info()->strides_in_bytes().y()) + (pool_start_y - pool_pad_top) * static_cast<int>(src->info()->strides_in_bytes().z());
78  const int in_x1_offset = (pool_start_x + 1 - pool_pad_left) * static_cast<int>(src->info()->strides_in_bytes().y()) + (pool_start_y - pool_pad_top) * static_cast<int>
79  (src->info()->strides_in_bytes().z());
80  const int in_x2_offset = (pool_start_x - pool_pad_left) * static_cast<int>(src->info()->strides_in_bytes().y()) + (pool_start_y + 1 - pool_pad_top) * static_cast<int>
81  (src->info()->strides_in_bytes().z());
82  const int in_x3_offset = (pool_start_x + 1 - pool_pad_left) * static_cast<int>(src->info()->strides_in_bytes().y()) + (pool_start_y + 1 - pool_pad_top) * static_cast<int>
83  (src->info()->strides_in_bytes().z());
84 
85  int x_off = window_start_x;
86  for(; x_off <= (window_end_x - window_step_x); x_off += window_step_x)
87  {
88  const auto in_x0_ptr = reinterpret_cast<const float *>(in.ptr() + in_x0_offset);
89  const auto in_x1_ptr = reinterpret_cast<const float *>(in.ptr() + in_x1_offset);
90  const auto in_x2_ptr = reinterpret_cast<const float *>(in.ptr() + in_x2_offset);
91  const auto in_x3_ptr = reinterpret_cast<const float *>(in.ptr() + in_x3_offset);
92  const auto v_x0 = vld1q_f32(in_x0_ptr + x_off);
93  const auto v_x1 = vld1q_f32(in_x1_ptr + x_off);
94  const auto v_x2 = vld1q_f32(in_x2_ptr + x_off);
95  const auto v_x3 = vld1q_f32(in_x3_ptr + x_off);
96  vres = vmaxq_f32(vmaxq_f32(v_x2, v_x3), vmaxq_f32(v_x0, v_x1));
97  // Store result
98  vst1q_f32(reinterpret_cast<float *>(out.ptr()) + x_off, vres);
99 
100  const uint32_t offset_base = offset_no_padding<float>(in.offset(), id, *src->info(), pool_stride_x, pool_stride_y, DataLayout::NHWC);
101  const uint32_t offset_x0 = (uint32_t)offset_base / sizeof(float) + x_off;
102  const uint32_t offset_x1 = (uint32_t)offset_x0 + in_stride_y / sizeof(float) - pad_horizontal;
103  const uint32_t offset_x2 = (uint32_t)offset_x0 + in_stride_z / sizeof(float) - pad_horizontal * src->info()->tensor_shape()[1];
104  const uint32_t offset_x3 = (uint32_t)offset_x2 + in_stride_y / sizeof(float) - pad_horizontal;
105  const uint32x4_t voffset_x0 = { offset_x0, offset_x0 + 1, offset_x0 + 2, offset_x0 + 3 };
106  const uint32x4_t voffset_x1 = { offset_x1, offset_x1 + 1, offset_x1 + 2, offset_x1 + 3 };
107  const uint32x4_t voffset_x2 = { offset_x2, offset_x2 + 1, offset_x2 + 2, offset_x2 + 3 };
108  const uint32x4_t voffset_x3 = { offset_x3, offset_x3 + 1, offset_x3 + 2, offset_x3 + 3 };
109  const uint32x4_t tmp_indices0 = vbslq_u32(vcgeq_f32(v_x0, v_x1), voffset_x0, voffset_x1);
110  const uint32x4_t tmp_indices1 = vbslq_u32(vcgeq_f32(v_x2, v_x3), voffset_x2, voffset_x3);
111  const uint32x4_t tmp_indices2 = vbslq_u32(vcgeq_f32(vmaxq_f32(v_x0, v_x1), vmaxq_f32(v_x2, v_x3)), tmp_indices0, tmp_indices1);
112 
113  // Store indices
114  vst1q_u32(reinterpret_cast<uint32_t *>(indices.ptr()) + x_off, tmp_indices2);
115  }
116 
117  // Left-overs loop
118  for(; x_off < window_end_x; ++x_off)
119  {
120  const auto x0 = *(reinterpret_cast<const float *>(in.ptr() + in_x0_offset) + x_off);
121  const auto x1 = *(reinterpret_cast<const float *>(in.ptr() + in_x1_offset) + x_off);
122  const auto x2 = *(reinterpret_cast<const float *>(in.ptr() + in_x2_offset) + x_off);
123  const auto x3 = *(reinterpret_cast<const float *>(in.ptr() + in_x3_offset) + x_off);
124  res = std::max(std::max(x2, x3), std::max(x0, x1));
125 
126  // Store result
127  *(reinterpret_cast<float *>(out.ptr()) + x_off) = res;
128 
129  const uint32_t offset_base = offset_no_padding<float>(in.offset(), id, *src->info(), pool_stride_x, pool_stride_y, DataLayout::NHWC);
130  const uint32_t offset_x0 = (uint32_t)offset_base / sizeof(float) + x_off;
131  const uint32_t offset_x1 = (uint32_t)offset_x0 + in_stride_y / sizeof(float) - pad_horizontal;
132  const uint32_t offset_x2 = (uint32_t)offset_x0 + in_stride_z / sizeof(float) - pad_horizontal * src->info()->tensor_shape()[1];
133  const uint32_t offset_x3 = (uint32_t)offset_x2 + in_stride_y / sizeof(float) - pad_horizontal;
134  const uint32_t tmp_idx0 = (x0 >= x1) ? offset_x0 : offset_x1;
135  const uint32_t tmp_idx1 = (x2 >= x3) ? offset_x2 : offset_x3;
136  const uint32_t tmp_idx2 = (std::max(x0, x1) >= std::max(x2, x3)) ? tmp_idx0 : tmp_idx1;
137 
138  // Store indices
139  *(reinterpret_cast<uint32_t *>(indices.ptr()) + x_off) = tmp_idx2;
140  }
141  },
142  in, out, indices);
143 }
144 }
145 
146 void poolingMxN_fp32_neon_nhwc(const ITensor *src, ITensor *dst0, ITensor *dst1, PoolingLayerInfo &pool_info, const Window &window_src, const Window &window)
147 {
148  if(pool_info.pool_size == Size2D(2, 2) && pool_info.pool_type == PoolingType::MAX && dst1)
149  {
150  pooling2_f32_maxpool_indices(src, dst0, dst1, pool_info, window_src, window);
151  }
152  else
153  {
154  const int window_start_x = window.x().start();
155  const int window_end_x = window.x().end();
156  const int window_step_x = 4;
157 
158  Window window_out = window;
159  window_out.set(Window::DimX, Window::Dimension(0, 1, 1));
160 
161  Iterator in(src, window_src);
162  Iterator out(dst0, window_out);
163 
164  const int pool_size_x = pool_info.is_global_pooling ? src->info()->tensor_shape().y() : pool_info.pool_size.width;
165  const int pool_size_y = pool_info.is_global_pooling ? src->info()->tensor_shape().z() : pool_info.pool_size.height;
166  const int pool_pad_right = pool_info.pad_stride_info.pad_right();
167  const int pool_pad_top = pool_info.pad_stride_info.pad_top();
168  const int pool_pad_left = pool_info.pad_stride_info.pad_left();
169  const int pool_pad_bottom = pool_info.pad_stride_info.pad_bottom();
170  int pool_stride_x = 0;
171  int pool_stride_y = 0;
172  std::tie(pool_stride_x, pool_stride_y) = pool_info.pad_stride_info.stride();
173  const int upper_bound_w = src->info()->dimension(1) + (pool_info.exclude_padding ? 0 : pool_pad_right);
174  const int upper_bound_h = src->info()->dimension(2) + (pool_info.exclude_padding ? 0 : pool_pad_bottom);
175 
176  float32x4_t vres;
177 
178  execute_window_loop(window_out, [&](const Coordinates & id)
179  {
180  const int idx_width = id.y() * pool_stride_x;
181  const int idx_height = id.z() * pool_stride_y;
182  const int pool_limit_y = pool_pad_top - idx_height;
183  const int pool_limit_x = pool_pad_left - idx_width;
184 
185  const int pool_start_y = std::max(0, window_src.z().start() + pool_limit_y);
186  const int pool_end_y = std::min(pool_size_y, window_src.z().end() + pool_limit_y);
187  const int pool_start_x = std::max(0, window_src.y().start() + pool_limit_x);
188  const int pool_end_x = std::min(pool_size_x, window_src.y().end() + pool_limit_x);
189 
190  int x_off = window_start_x;
191  for(; x_off <= (window_end_x - window_step_x); x_off += window_step_x)
192  {
193  if(pool_info.pool_type != PoolingType::MAX)
194  {
195  // Calculate scale
196  const float scale = calculate_avg_scale(pool_info.exclude_padding, DataLayout::NHWC, id, pool_size_x, pool_size_y, upper_bound_w, upper_bound_h, pool_pad_left, pool_pad_top, pool_stride_x,
197  pool_stride_y);
198  const float32x4_t scale_v = vdupq_n_f32(scale);
199 
200  // Perform pooling
201  vres = vdupq_n_f32(0.0f);
202 
203  for(int y = pool_start_y; y < pool_end_y; ++y)
204  {
205  for(int x = pool_start_x; x < pool_end_x; ++x)
206  {
207  const float32x4_t data = vld1q_f32(reinterpret_cast<const float *>(in.ptr() + (x - pool_pad_left) * static_cast<int>(src->info()->strides_in_bytes().y()) + (y - pool_pad_top) * static_cast<int>
208  (src->info()->strides_in_bytes().z())) + x_off);
209 
210  // Get power of 2 in case of l2 pooling and accumulate
211  if(pool_info.pool_type == PoolingType::L2)
212  {
213  vres = vmlaq_f32(vres, data, data);
214  }
215  else
216  {
217  vres = vaddq_f32(vres, data);
218  }
219  }
220  }
221  // Divide by scale
222  vres = vmulq_f32(vres, scale_v);
223  }
224  else
225  {
226  vres = vdupq_n_f32(std::numeric_limits<float>::lowest());
227  for(int y = pool_start_y; y < pool_end_y; ++y)
228  {
229  for(int x = pool_start_x; x < pool_end_x; ++x)
230  {
231  const float32x4_t data = vld1q_f32(reinterpret_cast<const float *>(in.ptr() + (x - pool_pad_left) * static_cast<int>(src->info()->strides_in_bytes().y()) + (y - pool_pad_top) * static_cast<int>
232  (src->info()->strides_in_bytes().z())) + x_off);
233  vres = vmaxq_f32(vres, data);
234  }
235  }
236  }
237 
238  // Calculate square-root in case of l2 pooling
239  if(pool_info.pool_type == PoolingType::L2)
240  {
241  float32x4_t l2_res = { static_cast<float>(sqrt(vgetq_lane_f32(vres, 0))),
242  static_cast<float>(sqrt(vgetq_lane_f32(vres, 1))),
243  static_cast<float>(sqrt(vgetq_lane_f32(vres, 2))),
244  static_cast<float>(sqrt(vgetq_lane_f32(vres, 3)))
245  };
246  vres = l2_res;
247  }
248 
249  // Store result
250  vst1q_f32(reinterpret_cast<float *>(out.ptr()) + x_off, vres);
251  }
252 
253  // Left-overs loop
254  for(; x_off < window_end_x; ++x_off)
255  {
256  float res = 0.0f;
257 
258  if(pool_info.pool_type != PoolingType::MAX)
259  {
260  // Calculate scale
261  const float scale = calculate_avg_scale(pool_info.exclude_padding, DataLayout::NHWC, id, pool_size_x, pool_size_y, upper_bound_w, upper_bound_h, pool_pad_left, pool_pad_top, pool_stride_x,
262  pool_stride_y);
263 
264  for(int y = pool_start_y; y < pool_end_y; ++y)
265  {
266  for(int x = pool_start_x; x < pool_end_x; ++x)
267  {
268  const float data = *(reinterpret_cast<const float *>(in.ptr() + (x - pool_pad_left) * static_cast<int>(src->info()->strides_in_bytes().y()) + (y - pool_pad_top) * static_cast<int>
269  (src->info()->strides_in_bytes().z())) + x_off);
270 
271  // Get power of 2 in case of l2 pooling and accumulate
272  if(pool_info.pool_type == PoolingType::L2)
273  {
274  res += data * data;
275  }
276  else
277  {
278  res += data;
279  }
280  }
281  }
282 
283  // Divide by scale
284  res *= scale;
285  }
286  else
287  {
289  for(int y = pool_start_y; y < pool_end_y; ++y)
290  {
291  for(int x = pool_start_x; x < pool_end_x; ++x)
292  {
293  const float data = *(reinterpret_cast<const float *>(in.ptr() + (x - pool_pad_left) * static_cast<int>(src->info()->strides_in_bytes().y()) + (y - pool_pad_top) * static_cast<int>
294  (src->info()->strides_in_bytes().z())) + x_off);
295  res = std::max(res, data);
296  }
297  }
298  }
299 
300  // Calculate square-root in case of l2 pooling
301  if(pool_info.pool_type == PoolingType::L2)
302  {
303  res = std::sqrt(res);
304  }
305 
306  // Store result
307  *(reinterpret_cast<float *>(out.ptr()) + x_off) = res;
308  }
309  },
310  in, out);
311  }
312 }
313 } // namespace cpu
314 } // namespace arm_compute
virtual size_t dimension(size_t index) const =0
Return the size of the requested dimension.
void poolingMxN_fp32_neon_nhwc(const ITensor *src, ITensor *dst0, ITensor *dst1, PoolingLayerInfo &pool_info, const Window &window_src, const Window &window)
Definition: fp32.cpp:146
Describe one of the image&#39;s dimensions with a start, end and step.
Definition: Window.h:77
unsigned int pad_top() const
Get the top padding.
Definition: Types.h:740
constexpr const Dimension & z() const
Alias to access the third dimension of the window.
Definition: Window.h:163
Interface for CPU tensor.
Definition: ITensor.h:36
SimpleTensor< float > src
Definition: DFT.cpp:155
Copyright (c) 2017-2021 Arm Limited.
size_t height
Height of the image region or rectangle.
Definition: Size2D.h:91
static constexpr size_t DimX
Alias for dimension 0 also known as X dimension.
Definition: Window.h:43
virtual const TensorShape & tensor_shape() const =0
Size for each dimension of the tensor.
T z() const
Alias to access the size of the third dimension.
Definition: Dimensions.h:97
Coordinates of an item.
Definition: Coordinates.h:37
std::pair< unsigned int, unsigned int > stride() const
Get the stride.
Definition: Types.h:704
Pooling Layer Information struct.
Definition: Types.h:1173
virtual ITensorInfo * info() const =0
Interface to be implemented by the child class to return the tensor&#39;s metadata.
unsigned int pad_right() const
Get the right padding.
Definition: Types.h:735
constexpr uint8_t * ptr() const
Return a pointer to the current pixel.
Definition: Helpers.inl:139
void set(size_t dimension, const Dimension &dim)
Set the values of a given dimension.
Definition: Window.inl:49
float calculate_avg_scale(bool exclude_padding, DataLayout data_layout, const Coordinates &id, const int pool_size_x, const int pool_size_y, const int upper_bound_w, const int upper_bound_h, const int pad_x, const int pad_y, const int stride_x, const int stride_y)
Definition: quantized.h:162
PadStrideInfo pad_stride_info
Definition: Types.h:1261
size_t width
Width of the image region or rectangle.
Definition: Size2D.h:90
Class for specifying the size of an image or rectangle.
Definition: Size2D.h:34
Num samples, height, width, channels.
int pool_stride_x
constexpr const Dimension & y() const
Alias to access the second dimension of the window.
Definition: Window.h:154
void execute_window_loop(const Window &w, L &&lambda_function, Ts &&... iterators)
Iterate through the passed window, automatically adjusting the iterators and calling the lambda_funct...
Definition: Helpers.inl:77
T y() const
Alias to access the size of the second dimension.
Definition: Dimensions.h:92
virtual const Strides & strides_in_bytes() const =0
The strides in bytes for accessing each dimension of the tensor.
constexpr int end() const
Return the end of the dimension.
Definition: Window.h:99
unsigned int pad_bottom() const
Get the bottom padding.
Definition: Types.h:745
Iterator updated by execute_window_loop for each window element.
Definition: Helpers.h:46
unsigned int pad_left() const
Get the left padding.
Definition: Types.h:730
constexpr int start() const
Return the start of the dimension.
Definition: Window.h:94
Describe a multidimensional execution window.
Definition: Window.h:39
constexpr const Dimension & x() const
Alias to access the first dimension of the window.
Definition: Window.h:145