Compute Library
 21.08
CpuDepthwiseConv2dNativeKernel.cpp
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2019-2021 Arm Limited.
3  *
4  * SPDX-License-Identifier: MIT
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a copy
7  * of this software and associated documentation files (the "Software"), to
8  * deal in the Software without restriction, including without limitation the
9  * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10  * sell copies of the Software, and to permit persons to whom the Software is
11  * furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in all
14  * copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22  * SOFTWARE.
23  */
25 
30 #include "src/core/CPP/Validate.h"
36 
37 namespace arm_compute
38 {
39 namespace cpu
40 {
41 namespace kernels
42 {
43 namespace
44 {
45 constexpr auto data_layout = DataLayout::NHWC;
46 const size_t width_idx = get_data_layout_dimension_index(data_layout, DataLayoutDimension::WIDTH);
47 const size_t height_idx = get_data_layout_dimension_index(data_layout, DataLayoutDimension::HEIGHT);
48 const size_t channel_idx = get_data_layout_dimension_index(data_layout, DataLayoutDimension::CHANNEL);
49 
50 constexpr auto dim_manual_loop = Window::Dimension(0, 0, 0);
51 constexpr auto dim_single_unit_step = Window::Dimension(0, 1, 1);
52 constexpr size_t vector_size = 8;
53 
54 struct DepthwiseConvolutionRunInfo
55 {
57  const uint32_t x_start;
58  const uint32_t x_end;
59  const uint32_t x_step;
60  const uint32_t x_leftover_start;
61  const size_t input_stride_y;
62  const size_t input_stride_z;
63  const size_t input_max_offset;
64  const size_t weights_width;
65  const size_t weights_height;
66  const size_t weights_stride_y;
67  const size_t weights_stride_z;
68  const size_t conv_stride_x;
69  const size_t conv_stride_y;
70  const size_t conv_pad_left;
71  const size_t conv_pad_top;
72  const size_t input_height;
73  const size_t input_width;
74  const size_t input_depth;
75 
76  DepthwiseConvolutionRunInfo(const ITensorInfo &input, const ITensorInfo &weights, const PadStrideInfo &conv_info, const Window &w, uint32_t depth_multiplier = 1) // NOLINT
77  : num_read_elements_per_iteration((depth_multiplier == 1 ? (vector_size / element_size_from_data_type(input.data_type())) : 1)),
78  x_start(w.x().start()),
79  x_end(w.x().end()),
80  x_step(static_cast<uint32_t>(num_read_elements_per_iteration * depth_multiplier)),
81  x_leftover_start(std::max(static_cast<int32_t>(w.x().end()) - static_cast<int32_t>(x_step) + 1, int32_t(0))),
82  input_stride_y(input.strides_in_bytes().y()),
83  input_stride_z(input.strides_in_bytes().z()),
84  input_max_offset(input.strides_in_bytes().z() * input.dimension(height_idx) - (input.padding().bottom + input.padding().top) * input.strides_in_bytes().y()),
85  weights_width(weights.dimension(width_idx)),
86  weights_height(weights.dimension(height_idx)),
87  weights_stride_y(weights.strides_in_bytes().y()),
88  weights_stride_z(weights.strides_in_bytes().z()),
89  conv_stride_x(conv_info.stride().first),
90  conv_stride_y(conv_info.stride().second),
91  conv_pad_left(conv_info.pad_left()),
92  conv_pad_top(conv_info.pad_top()),
93  input_height(input.dimension(height_idx)),
94  input_width(input.dimension(width_idx)),
95  input_depth(input.dimension(channel_idx))
96  {
97  }
98 };
99 
100 inline int32x4_t saturating_doubling_high_mul(const int32x4_t &a, const int32_t &b)
101 {
102  return vqrdmulhq_n_s32(a, b);
103 }
104 
105 inline int32_t saturating_doubling_high_mul(const int32_t &a, const int32_t &b)
106 {
107  return vget_lane_s32(vqrdmulh_n_s32(vdup_n_s32(a), b), 0);
108 }
109 
110 inline int32x4_t rounding_divide_by_exp2(const int32x4_t &x, const int exponent)
111 {
112  const int32x4_t shift = vdupq_n_s32(-exponent);
113  const int32x4_t fixup = vshrq_n_s32(vandq_s32(x, shift), 31);
114  const int32x4_t fixed = vqaddq_s32(x, fixup);
115  return vrshlq_s32(fixed, shift);
116 }
117 
118 inline int32x2_t rounding_divide_by_exp2(const int32x2_t &x, const int exponent)
119 {
120  const int32x2_t shift = vdup_n_s32(-exponent);
121  const int32x2_t fixup = vshr_n_s32(vand_s32(x, shift), 31);
122  const int32x2_t fixed = vqadd_s32(x, fixup);
123  return vrshl_s32(fixed, shift);
124 }
125 
126 inline int32_t rounding_divide_by_exp2(const int32_t &x, const int exponent)
127 {
128  const int32x2_t xs = vdup_n_s32(x);
129  return vget_lane_s32(rounding_divide_by_exp2(xs, exponent), 0);
130 }
131 
132 inline bool is_valid_input_region(int32_t base_w, uint32_t base_h, uint32_t w, uint32_t h, const DepthwiseConvolutionRunInfo &run_info, const Size2D &dilation)
133 {
134  const int32_t current_h = base_h + h * dilation.y();
135  const bool is_valid_h = current_h >= 0 && current_h < static_cast<int32_t>(run_info.input_height);
136 
137  const int32_t current_w = base_w + w * dilation.x();
138  const bool is_valid_w = current_w >= 0 && current_w < static_cast<int32_t>(run_info.input_width);
139 
140  return is_valid_h && is_valid_w;
141 }
142 
143 template <typename T>
144 void depthwise_loop_multiplier1_fp(const ITensor *src, const ITensor *weights, const ITensor *biases, ITensor *dst, const PadStrideInfo &conv_info,
145  const Size2D &dilation, const Window &window, bool has_biases)
146 {
147  constexpr auto element_per_vector = vector_size / sizeof(T);
149  using TagType = typename wrapper::traits::neon_vector<T, element_per_vector>::tag_type;
150 
151  const auto run_info = DepthwiseConvolutionRunInfo(*src->info(), *weights->info(), conv_info, window);
152 
153  const VectorType zero_vector = wrapper::vdup_n(static_cast<T>(0), TagType{});
154 
155  Window execution_window = window;
156  execution_window.set(Window::DimX, dim_single_unit_step);
157 
158  Window win_input = window;
159  win_input.set(Window::DimX, dim_manual_loop);
160  win_input.set(Window::DimY, dim_manual_loop);
161  win_input.set(Window::DimZ, dim_manual_loop);
162 
163  Window win_weights = win_input;
164  win_weights.set(Window::DimW, dim_manual_loop);
165 
166  Window win_output = window;
167  win_output.set(Window::DimX, dim_manual_loop);
168 
169  Iterator input_it(src, win_input);
170  Iterator weights_it(weights, win_weights);
171  Iterator output_it(dst, win_output);
172  Iterator biases_it{};
173 
174  if(has_biases)
175  {
176  biases_it = Iterator(biases, win_weights);
177  }
178 
179  execute_window_loop(execution_window, [&](const Coordinates & id)
180  {
181  const int32_t input_y = id.y() * run_info.conv_stride_x - run_info.conv_pad_left;
182  const int32_t input_z = id.z() * run_info.conv_stride_y - run_info.conv_pad_top;
183  const int64_t base_input_offset = input_y * run_info.input_stride_y + input_z * run_info.input_stride_z;
184 
185  auto const base_weights_ptr = weights_it.ptr();
186  uint32_t x = run_info.x_start;
187 
188  for(; x < run_info.x_leftover_start; x += run_info.x_step)
189  {
190  VectorType acc = zero_vector;
191  auto weights_ptr = base_weights_ptr;
192  int64_t input_offset = base_input_offset;
193 
194  for(uint32_t h = 0; h < run_info.weights_height; ++h)
195  {
196  int64_t offs = input_offset + x * sizeof(T);
197  for(uint32_t w = 0; w < run_info.weights_width; ++w)
198  {
199  const bool is_valid_region = is_valid_input_region(input_y, input_z, w, h, run_info, dilation);
200  const auto input_vals = is_valid_region ?
201  wrapper::vload(reinterpret_cast<T *>(input_it.ptr() + std::min(static_cast<size_t>(offs), run_info.input_max_offset))) :
202  zero_vector;
203  const auto weights_vals = wrapper::vload(reinterpret_cast<T *>(weights_ptr + w * run_info.weights_stride_y) + x);
204  acc = wrapper::vmla(acc, weights_vals, input_vals);
205 
206  offs += dilation.x() * run_info.input_stride_y;
207  }
208 
209  weights_ptr += run_info.weights_stride_z;
210  input_offset += dilation.y() * run_info.input_stride_z;
211  }
212 
213  if(has_biases)
214  {
215  const auto biases_vals = wrapper::vload(reinterpret_cast<T *>(biases_it.ptr()) + x);
216  acc = wrapper::vadd(acc, biases_vals);
217  }
218 
219  wrapper::vstore(reinterpret_cast<T *>(output_it.ptr()) + x, acc);
220  }
221 
222  for(; x < run_info.x_end; ++x)
223  {
224  auto acc_scalar = T{ 0 };
225  auto weights_ptr = base_weights_ptr;
226  int64_t input_offset = base_input_offset;
227 
228  for(size_t h = 0; h < run_info.weights_height; ++h)
229  {
230  int64_t offs = input_offset + x * sizeof(T);
231  for(size_t w = 0; w < run_info.weights_width; ++w)
232  {
233  const bool is_valid_region = is_valid_input_region(input_y, input_z, w, h, run_info, dilation);
234  const auto input_vals = is_valid_region ? *reinterpret_cast<T *>(input_it.ptr() + std::min(static_cast<size_t>(offs), run_info.input_max_offset)) : 0;
235  const auto weights_vals = *(reinterpret_cast<T *>(weights_ptr + w * run_info.weights_stride_y) + x);
236 
237  acc_scalar += (input_vals * weights_vals);
238 
239  offs += dilation.x() * run_info.input_stride_y;
240  }
241 
242  weights_ptr += run_info.weights_stride_z;
243  input_offset += dilation.y() * run_info.input_stride_z;
244  }
245 
246  if(has_biases)
247  {
248  const auto biases_vals = *(reinterpret_cast<T *>(biases_it.ptr()) + x);
249  acc_scalar += biases_vals;
250  }
251  *(reinterpret_cast<T *>(output_it.ptr()) + x) = acc_scalar;
252  }
253  },
254  input_it, weights_it, biases_it, output_it);
255 }
256 
257 template <typename T>
258 void depthwise_loop_generic_fp(const ITensor *src, const ITensor *weights, const ITensor *biases, ITensor *dst, const PadStrideInfo &conv_info,
259  const Size2D &dilation, unsigned int depth_multiplier, const Window &window, bool has_biases)
260 {
261  const auto run_info = DepthwiseConvolutionRunInfo(*src->info(), *weights->info(), conv_info, window, depth_multiplier);
262 
263  Window execution_window = window;
264  execution_window.set(Window::DimX, Window::Dimension(0, run_info.input_depth, 1));
265 
266  Window win_input = execution_window;
267  win_input.set(Window::DimX, Window::Dimension(0, run_info.input_depth, 1));
268  win_input.set(Window::DimY, dim_manual_loop);
269  win_input.set(Window::DimZ, dim_manual_loop);
270 
271  Window win_weights = window;
272  win_weights.set_dimension_step(Window::DimX, run_info.x_step);
273  win_weights.set(Window::DimY, dim_manual_loop);
274  win_weights.set(Window::DimZ, dim_manual_loop);
275  win_weights.set(Window::DimW, dim_manual_loop);
276 
277  Window win_output = window;
278  win_output.set_dimension_step(Window::DimX, run_info.x_step);
279 
280  Iterator input_it(src, win_input);
281  Iterator weights_it(weights, win_weights);
282  Iterator output_it(dst, win_output);
283  Iterator biases_it{};
284 
285  if(has_biases)
286  {
287  biases_it = Iterator(biases, win_weights);
288  }
289 
290  execute_window_loop(execution_window, [&](const Coordinates & id)
291  {
292  std::vector<T> acc(depth_multiplier, static_cast<T>(0));
293 
294  const int input_y = id.y() * run_info.conv_stride_x - run_info.conv_pad_left;
295  const int input_z = id.z() * run_info.conv_stride_y - run_info.conv_pad_top;
296  int input_offset = input_y * run_info.input_stride_y + input_z * run_info.input_stride_z;
297 
298  auto weights_ptr = weights_it.ptr();
299  for(size_t h = 0; h < run_info.weights_height; ++h)
300  {
301  int offs = input_offset;
302  for(size_t w = 0; w < run_info.weights_width; ++w)
303  {
304  const bool is_valid_region = is_valid_input_region(input_y, input_z, w, h, run_info, dilation);
305  const auto input_val = is_valid_region ? *(reinterpret_cast<T *>(input_it.ptr() + std::min(static_cast<size_t>(offs), run_info.input_max_offset))) : T(0);
306 
307  for(size_t m = 0; m < depth_multiplier; ++m)
308  {
309  const auto weights_val = *(reinterpret_cast<T *>(weights_ptr + m * sizeof(T) + w * run_info.weights_stride_y));
310  acc.at(m) = support::cpp11::fma(weights_val, input_val, acc.at(m));
311  }
312 
313  offs += dilation.x() * run_info.input_stride_y;
314  }
315 
316  weights_ptr += run_info.weights_stride_z;
317  input_offset += dilation.y() * run_info.input_stride_z;
318  }
319 
320  if(has_biases)
321  {
322  for(size_t m = 0; m < depth_multiplier; ++m)
323  {
324  const auto biases_val = *(reinterpret_cast<T *>(biases_it.ptr() + m * sizeof(T)));
325  *(reinterpret_cast<T *>(output_it.ptr() + m * sizeof(T))) = acc.at(m) + biases_val;
326  }
327  }
328  else
329  {
330  for(size_t m = 0; m < depth_multiplier; ++m)
331  {
332  *(reinterpret_cast<T *>(output_it.ptr() + m * sizeof(T))) = acc.at(m);
333  }
334  }
335  },
336  input_it, weights_it, biases_it, output_it);
337 }
338 
339 template <typename T, typename TW>
340 void depthwise_loop_multiplier1_quantized(const ITensor *src, const ITensor *weights, const ITensor *biases, ITensor *dst, const PadStrideInfo &conv_info,
341  const Size2D &dilation, std::vector<int> output_multiplier, std::vector<int> output_shift, const Window &window, bool has_biases) // NOLINT
342 {
343  ARM_COMPUTE_UNUSED(output_multiplier, output_shift);
344  constexpr auto element_per_vector = vector_size / sizeof(T);
346  using TagType = typename wrapper::traits::neon_vector<T, element_per_vector>::tag_type;
347  using AccType = int32_t;
348  using AccArrayType = std::array<AccType, element_per_vector>;
349 
350  const auto out_of_bound_value = PixelValue(static_cast<uint64_t>(0), src->info()->data_type(), src->info()->quantization_info()).get<T>();
351  const auto out_of_bound_vector = wrapper::vdup_n(static_cast<T>(out_of_bound_value), TagType{});
352 
353  const auto run_info = DepthwiseConvolutionRunInfo(*src->info(), *weights->info(), conv_info, window);
354 
355  const int32_t input_qoffset = src->info()->quantization_info().uniform().offset;
356  const int32_t weights_qoffset = weights->info()->quantization_info().uniform().offset;
357  const int32_t output_qoffset = dst->info()->quantization_info().uniform().offset;
358  const int32_t k_offset = run_info.weights_width * run_info.weights_height * input_qoffset * weights_qoffset;
359 
360  Window execution_window = window;
361  execution_window.set(Window::DimX, dim_single_unit_step);
362 
363  Window win_input = window;
364  win_input.set(Window::DimX, dim_manual_loop);
365  win_input.set(Window::DimY, dim_manual_loop);
366  win_input.set(Window::DimZ, dim_manual_loop);
367 
368  Window win_weights = win_input;
369  win_weights.set(Window::DimW, dim_manual_loop);
370 
371  Window win_output = window;
372  win_output.set(Window::DimX, dim_manual_loop);
373 
374  Iterator input_it(src, win_input);
375  Iterator weights_it(weights, win_weights);
376  Iterator output_it(dst, win_output);
377  Iterator biases_it{};
378 
379  if(has_biases)
380  {
381  biases_it = Iterator(biases, win_weights);
382  }
383 
384  execute_window_loop(execution_window, [&](const Coordinates & id)
385  {
386  const int32_t input_y = id.y() * run_info.conv_stride_x - run_info.conv_pad_left;
387  const int32_t input_z = id.z() * run_info.conv_stride_y - run_info.conv_pad_top;
388  const int64_t base_input_offset = input_y * run_info.input_stride_y + input_z * run_info.input_stride_z;
389  auto const base_weights_ptr = weights_it.ptr();
390  size_t x = run_info.x_start;
391 
392  for(; x < run_info.x_leftover_start; x += run_info.x_step)
393  {
394  AccArrayType acc{};
395  AccArrayType in_sum{};
396  AccArrayType we_sum{};
397 
398  auto weights_ptr = base_weights_ptr;
399  auto input_offset = base_input_offset;
400 
401  for(size_t h = 0; h < run_info.weights_height; ++h)
402  {
403  int64_t offs = input_offset + x * sizeof(T);
404  for(size_t w = 0; w < run_info.weights_width; ++w)
405  {
406  const bool is_valid_region = is_valid_input_region(input_y, input_z, w, h, run_info, dilation);
407  const auto input_vals = is_valid_region ?
408  wrapper::vload(reinterpret_cast<T *>(input_it.ptr() + std::min(static_cast<size_t>(offs), run_info.input_max_offset))) :
409  out_of_bound_vector;
410  const auto weights_vals = wrapper::vload(reinterpret_cast<TW *>(weights_ptr + w * run_info.weights_stride_y) + x);
411 
412  for(size_t i = 0; i < element_per_vector; ++i)
413  {
414  acc.at(i) += input_vals[i] * weights_vals[i];
415  in_sum.at(i) += input_vals[i];
416  we_sum.at(i) += weights_vals[i];
417  }
418 
419  offs += dilation.x() * run_info.input_stride_y;
420  }
421 
422  weights_ptr += run_info.weights_stride_z;
423  input_offset += dilation.y() * run_info.input_stride_z;
424  }
425 
426  VectorType out_vals = wrapper::vdup_n(static_cast<T>(0), TagType{});
427  for(size_t i = 0; i < element_per_vector; ++i)
428  {
429  acc.at(i) -= in_sum.at(i) * weights_qoffset;
430  acc.at(i) -= we_sum.at(i) * input_qoffset;
431  acc.at(i) += k_offset;
432 
433  if(has_biases)
434  {
435  acc.at(i) += *(reinterpret_cast<int32_t *>(biases_it.ptr() + i * sizeof(int32_t)) + x);
436  }
437 
438  const int32_t out_mul = output_multiplier.at(x + i);
439  const int32_t out_shift = output_shift.at(x + i);
440  if(out_shift < 0)
441  {
442  acc.at(i) = saturating_doubling_high_mul(acc.at(i) * (1 << (-out_shift)), out_mul) + output_qoffset;
443  }
444  else
445  {
446  acc.at(i) = rounding_divide_by_exp2(saturating_doubling_high_mul(acc.at(i), out_mul), out_shift) + output_qoffset;
447  }
448  out_vals[i] = static_cast<T>(utility::clamp<AccType, T>(acc.at(i)));
449  }
450 
451  wrapper::vstore(reinterpret_cast<T *>(output_it.ptr()) + x, out_vals);
452  }
453 
454  // left-over
455  for(; x < run_info.x_end; ++x)
456  {
457  AccType acc = 0;
458  AccType in_sum = 0;
459  AccType we_sum = 0;
460 
461  auto weights_ptr = base_weights_ptr;
462  auto input_offset = base_input_offset;
463 
464  for(size_t h = 0; h < run_info.weights_height; ++h)
465  {
466  int64_t offs = input_offset + x * sizeof(T);
467  for(size_t w = 0; w < run_info.weights_width; ++w)
468  {
469  const bool is_valid_region = is_valid_input_region(input_y, input_z, w, h, run_info, dilation);
470  const auto input_val = is_valid_region ?
471  *reinterpret_cast<T *>(input_it.ptr() + std::min(static_cast<size_t>(offs), run_info.input_max_offset)) :
472  out_of_bound_value;
473  const auto weights_val = *(reinterpret_cast<TW *>(weights_ptr + w * run_info.weights_stride_y) + x);
474 
475  acc += input_val * weights_val;
476  in_sum += input_val;
477  we_sum += weights_val;
478 
479  offs += dilation.x() * run_info.input_stride_y;
480  }
481 
482  weights_ptr += run_info.weights_stride_z;
483  input_offset += dilation.y() * run_info.input_stride_z;
484  }
485 
486  T out_vals{ 0 };
487 
488  acc -= in_sum * weights_qoffset;
489  acc -= we_sum * input_qoffset;
490  acc += k_offset;
491 
492  if(has_biases)
493  {
494  acc += *(reinterpret_cast<int32_t *>(biases_it.ptr()) + x);
495  }
496 
497  const int32_t out_mul = output_multiplier.at(x);
498  const int32_t out_shift = output_shift.at(x);
499 
500  if(out_shift < 0)
501  {
502  acc = saturating_doubling_high_mul(acc * (1 << (-out_shift)), out_mul) + output_qoffset;
503  }
504  else
505  {
506  acc = rounding_divide_by_exp2(saturating_doubling_high_mul(acc, out_mul), out_shift) + output_qoffset;
507  }
508 
509  out_vals = static_cast<T>(utility::clamp<AccType, T>(acc));
510  *(reinterpret_cast<T *>(output_it.ptr()) + x) = out_vals;
511  }
512  },
513  input_it, weights_it, biases_it, output_it);
514 }
515 
516 template <typename T, typename TW>
517 void depthwise_loop_generic_quantized(const ITensor *src, const ITensor *weights, const ITensor *biases, ITensor *dst, const PadStrideInfo &conv_info,
518  const Size2D &dilation, unsigned int depth_multiplier, std::vector<int> output_multiplier, std::vector<int> output_shift, const Window &window, bool has_biases) // NOLINT
519 {
520  using AccType = int32_t;
521 
522  const auto run_info = DepthwiseConvolutionRunInfo(*src->info(), *weights->info(), conv_info, window, depth_multiplier);
523 
524  const auto out_of_bound_value = PixelValue(static_cast<uint64_t>(0), src->info()->data_type(), src->info()->quantization_info()).get<T>();
525 
526  const int32_t input_qoffset = src->info()->quantization_info().uniform().offset;
527  const int32_t weights_qoffset = weights->info()->quantization_info().uniform().offset;
528  const int32_t output_qoffset = dst->info()->quantization_info().uniform().offset;
529  const int32_t k_offset = run_info.weights_width * run_info.weights_height * input_qoffset * weights_qoffset;
530 
531  Window execution_window = window;
532  execution_window.set(Window::DimX, Window::Dimension(0, run_info.input_depth, 1));
533 
534  Window win_input = execution_window;
535  win_input.set(Window::DimY, dim_manual_loop);
536  win_input.set(Window::DimZ, dim_manual_loop);
537 
538  Window win_weights = window;
539  win_weights.set_dimension_step(Window::DimX, run_info.x_step);
540  win_weights.set(Window::DimY, dim_manual_loop);
541  win_weights.set(Window::DimZ, dim_manual_loop);
542  win_weights.set(Window::DimW, dim_manual_loop);
543 
544  Window win_output = window;
545  win_output.set_dimension_step(Window::DimX, run_info.x_step);
546 
547  Iterator input_it(src, win_input);
548  Iterator weights_it(weights, win_weights);
549  Iterator output_it(dst, win_output);
550  Iterator biases_it{};
551 
552  if(has_biases)
553  {
554  biases_it = Iterator(biases, win_weights);
555  }
556 
557  execute_window_loop(execution_window, [&](const Coordinates & id)
558  {
559  std::vector<AccType> acc(depth_multiplier, 0);
560  std::vector<AccType> we_sum(depth_multiplier, 0);
561  AccType in_sum = 0;
562 
563  const int32_t input_y = id.y() * run_info.conv_stride_x - run_info.conv_pad_left;
564  const int32_t input_z = id.z() * run_info.conv_stride_y - run_info.conv_pad_top;
565  int64_t input_offset = input_y * run_info.input_stride_y + input_z * run_info.input_stride_z;
566 
567  auto weights_ptr = weights_it.ptr();
568  for(size_t h = 0; h < run_info.weights_height; ++h)
569  {
570  int offs = input_offset;
571  for(size_t w = 0; w < run_info.weights_width; ++w)
572  {
573  const bool is_valid_region = is_valid_input_region(input_y, input_z, w, h, run_info, dilation);
574  const auto input_val = is_valid_region ? *(reinterpret_cast<T *>(input_it.ptr() + std::min(static_cast<size_t>(offs), run_info.input_max_offset))) : out_of_bound_value;
575 
576  for(size_t m = 0; m < depth_multiplier; ++m)
577  {
578  const auto weights_val = *(reinterpret_cast<TW *>(weights_ptr + m * sizeof(T) + w * run_info.weights_stride_y));
579  acc.at(m) += input_val * weights_val;
580 
581  we_sum.at(m) += weights_val;
582  }
583 
584  offs += dilation.x() * run_info.input_stride_y;
585  in_sum += input_val;
586  }
587 
588  weights_ptr += run_info.weights_stride_z;
589  input_offset += dilation.y() * run_info.input_stride_z;
590  }
591 
592  for(size_t m = 0; m < depth_multiplier; ++m)
593  {
594  acc.at(m) -= in_sum * weights_qoffset;
595  acc.at(m) -= we_sum.at(m) * input_qoffset;
596  acc.at(m) += k_offset;
597 
598  if(has_biases)
599  {
600  acc.at(m) += *(reinterpret_cast<int32_t *>(biases_it.ptr() + m * sizeof(int32_t)));
601  }
602 
603  const int32_t out_mul = output_multiplier.at(id.x() * depth_multiplier + m);
604  const int32_t out_shift = output_shift.at(id.x() * depth_multiplier + m);
605  if(out_shift < 0)
606  {
607  acc.at(m) = saturating_doubling_high_mul(acc.at(m) * (1 << (-out_shift)), out_mul) + output_qoffset;
608  }
609  else
610  {
611  acc.at(m) = rounding_divide_by_exp2(saturating_doubling_high_mul(acc.at(m), out_mul), out_shift) + output_qoffset;
612  }
613  *(reinterpret_cast<T *>(output_it.ptr() + m * sizeof(T))) = static_cast<T>(utility::clamp<AccType, T>(acc.at(m)));
614  }
615  },
616  input_it, weights_it, biases_it, output_it);
617 }
618 
619 template <typename T, typename TW>
620 void depthwise_loop_pow2_quantized_per_tensor(const ITensor *src, const ITensor *weights, const ITensor *biases, ITensor *dst, const PadStrideInfo &conv_info,
621  const Size2D &dilation, unsigned int depth_multiplier, std::vector<int> output_multiplier, std::vector<int> output_shift, const Window &window, bool has_biases) // NOLINT
622 {
623  constexpr int half_vec = vector_size / 2;
624 
625  using AccType = int32_t;
626  using AccVectorType = typename wrapper::traits::neon_vector<AccType, half_vec>::type;
627  using AccVectorTagType = typename wrapper::traits::neon_vector<AccType, half_vec>::tag_type;
628  using TagType = typename wrapper::traits::neon_vector<T, vector_size>::tag_type;
629 
630  const auto run_info = DepthwiseConvolutionRunInfo(*src->info(), *weights->info(), conv_info, window, depth_multiplier);
631 
632  const auto input_qoffset_vec = wrapper::vreinterpret(wrapper::vmovl(wrapper::vdup_n(static_cast<T>(src->info()->quantization_info().uniform().offset), TagType{})));
633  const auto weights_qoffset_vec = wrapper::vreinterpret(wrapper::vmovl(wrapper::vdup_n(static_cast<TW>(weights->info()->quantization_info().uniform().offset), TagType{})));
634  const auto output_qoffset_vec = wrapper::vdup_n(dst->info()->quantization_info().uniform().offset, arm_compute::wrapper::traits::vector_128_tag{});
635 
636  const auto lower = wrapper::vdup_n(static_cast<AccType>(std::numeric_limits<T>::lowest()), AccVectorTagType{});
637  const auto upper = wrapper::vdup_n(static_cast<AccType>(std::numeric_limits<T>::max()), AccVectorTagType{});
638  const auto zero = wrapper::vdup_n(static_cast<AccType>(0), AccVectorTagType{});
639 
640  const auto out_mul = output_multiplier.at(0);
641  const auto out_shift = output_shift.at(0);
642 
643  Window execution_window = window;
644  execution_window.set(Window::DimX, Window::Dimension(0, run_info.input_depth, 1));
645 
646  Window win_input = execution_window;
647  win_input.set(Window::DimY, dim_manual_loop);
648  win_input.set(Window::DimZ, dim_manual_loop);
649 
650  Window win_weights = window;
651  win_weights.set_dimension_step(Window::DimX, run_info.x_step);
652  win_weights.set(Window::DimY, dim_manual_loop);
653  win_weights.set(Window::DimZ, dim_manual_loop);
654  win_weights.set(Window::DimW, dim_manual_loop);
655 
656  Window win_output = window;
657  win_output.set_dimension_step(Window::DimX, run_info.x_step);
658 
659  Iterator input_it(src, win_input);
660  Iterator weights_it(weights, win_weights);
661  Iterator output_it(dst, win_output);
662  Iterator biases_it{};
663 
664  if(has_biases)
665  {
666  biases_it = Iterator(biases, win_weights);
667  }
668 
669  std::vector<AccVectorType> acc0(depth_multiplier / vector_size);
670  std::vector<AccVectorType> acc1(depth_multiplier / vector_size);
671 
672  execute_window_loop(execution_window, [&](const Coordinates & id)
673  {
674  std::fill(begin(acc0), end(acc0), zero);
675  std::fill(begin(acc1), end(acc1), zero);
676 
677  const int32_t input_y = id.y() * run_info.conv_stride_x - run_info.conv_pad_left;
678  const int32_t input_z = id.z() * run_info.conv_stride_y - run_info.conv_pad_top;
679  int64_t input_offset = input_y * run_info.input_stride_y + input_z * run_info.input_stride_z;
680 
681  auto weights_ptr = weights_it.ptr();
682  for(size_t h = 0; h < run_info.weights_height; ++h)
683  {
684  const int32_t current_h = input_z + h * dilation.y();
685  if(current_h >= 0 && current_h < static_cast<int32_t>(run_info.input_height))
686  {
687  int offs = input_offset;
688  for(size_t w = 0; w < run_info.weights_width; ++w)
689  {
690  const int32_t current_w = input_y + w * dilation.x();
691  if(current_w >= 0 && current_w < static_cast<int32_t>(run_info.input_width))
692  {
693  const auto input_8x8 = wrapper::vdup_n(*(reinterpret_cast<T *>(input_it.ptr() + std::min(static_cast<size_t>(offs), run_info.input_max_offset))), TagType{});
694  const auto input_s16x8 = wrapper::vreinterpret(wrapper::vmovl(input_8x8));
695  const auto input_no_offs = wrapper::vsub(input_s16x8, input_qoffset_vec);
696 
697  for(size_t m = 0, i = 0; m < depth_multiplier; m += vector_size, ++i)
698  {
699  const auto weights_8x8 = wrapper::vload(reinterpret_cast<TW *>(weights_ptr + m * sizeof(T) + w * run_info.weights_stride_y));
700  const auto weights_s16x8 = wrapper::vreinterpret(wrapper::vmovl(weights_8x8));
701  const auto weights_no_offs = wrapper::vsub(weights_s16x8, weights_qoffset_vec);
702 
703  acc0.at(i) = wrapper::vmlal(acc0.at(i), wrapper::vgetlow(input_no_offs), wrapper::vgetlow(weights_no_offs));
704  acc1.at(i) = wrapper::vmlal(acc1.at(i), wrapper::vgethigh(input_no_offs), wrapper::vgethigh(weights_no_offs));
705  }
706  }
707 
708  offs += dilation.x() * run_info.input_stride_y;
709  }
710  }
711 
712  weights_ptr += run_info.weights_stride_z;
713  input_offset += dilation.y() * run_info.input_stride_z;
714  }
715 
716  for(size_t m = 0, i = 0; m < depth_multiplier; m += vector_size, ++i)
717  {
718  if(has_biases)
719  {
720  const auto bias_val0 = wrapper::vloadq(reinterpret_cast<int32_t *>(biases_it.ptr() + m * sizeof(int32_t)));
721  const auto bias_val1 = wrapper::vloadq(reinterpret_cast<int32_t *>(biases_it.ptr() + (m + half_vec) * sizeof(int32_t)));
722 
723  acc0.at(i) = wrapper::vadd(acc0.at(i), bias_val0);
724  acc1.at(i) = wrapper::vadd(acc1.at(i), bias_val1);
725  }
726 
727  if(out_shift < 0)
728  {
729  acc0.at(i) = wrapper::vadd(saturating_doubling_high_mul(acc0.at(i) * (1 << (-out_shift)), out_mul), output_qoffset_vec);
730  acc1.at(i) = wrapper::vadd(saturating_doubling_high_mul(acc1.at(i) * (1 << (-out_shift)), out_mul), output_qoffset_vec);
731  }
732  else
733  {
734  acc0.at(i) = wrapper::vadd(rounding_divide_by_exp2(saturating_doubling_high_mul(acc0.at(i), out_mul), out_shift), output_qoffset_vec);
735  acc1.at(i) = wrapper::vadd(rounding_divide_by_exp2(saturating_doubling_high_mul(acc1.at(i), out_mul), out_shift), output_qoffset_vec);
736  }
737 
738  acc0.at(i) = wrapper::vmin(wrapper::vmax(acc0.at(i), lower), upper);
739  acc1.at(i) = wrapper::vmin(wrapper::vmax(acc1.at(i), lower), upper);
740 
741  const auto out_val = wrapper::vcombine(wrapper::vmovn(acc0.at(i)),
742  wrapper::vmovn(acc1.at(i)));
743 
744  if(std::is_same<T, uint8_t>::value)
745  {
746  wrapper::vstore(reinterpret_cast<uint8_t *>(output_it.ptr() + m * sizeof(uint8_t)), wrapper::vqmovn(vreinterpretq_u16_s16(out_val)));
747  }
748  else
749  {
750  wrapper::vstore(reinterpret_cast<int8_t *>(output_it.ptr() + m * sizeof(int8_t)), wrapper::vqmovn(out_val));
751  }
752  }
753  },
754  input_it, weights_it, biases_it, output_it);
755 }
756 
757 Status validate_arguments(const ITensorInfo *src, const ITensorInfo *weights, const ITensorInfo *biases, const ITensorInfo *dst, const ConvolutionInfo &info)
758 {
759  ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(src, weights, dst);
761  ARM_COMPUTE_RETURN_ERROR_ON(src->data_layout() == DataLayout::UNKNOWN);
763  ARM_COMPUTE_RETURN_ERROR_ON(info.depth_multiplier == 0);
764  ARM_COMPUTE_RETURN_ERROR_ON(weights->dimension(1) + (weights->dimension(1) - 1) * (info.dilation.x() - 1) > src->dimension(1) + info.pad_stride_info.pad_left() + info.pad_stride_info.pad_right());
765  ARM_COMPUTE_RETURN_ERROR_ON(weights->dimension(2) + (weights->dimension(2) - 1) * (info.dilation.y() - 1) > src->dimension(2) + info.pad_stride_info.pad_top() + info.pad_stride_info.pad_bottom());
766  ARM_COMPUTE_RETURN_ERROR_ON((src->dimension(0) * info.depth_multiplier) != weights->dimension(0));
767  ARM_COMPUTE_RETURN_ERROR_ON((info.dilation.x() < 1) || (info.dilation.y() < 1));
768  ARM_COMPUTE_RETURN_ERROR_ON((info.pad_stride_info.stride().first < 1) || (info.pad_stride_info.stride().second < 1));
769 
770  if(is_data_type_quantized_per_channel(weights->data_type()))
771  {
773  ARM_COMPUTE_RETURN_ERROR_ON(weights->dimension(0) != weights->quantization_info().scale().size());
774  }
775  else
776  {
778  }
779 
780  if(biases != nullptr)
781  {
782  ARM_COMPUTE_RETURN_ERROR_ON(biases->num_dimensions() > 1);
783  ARM_COMPUTE_RETURN_ERROR_ON(biases->dimension(0) != weights->dimension(0));
784 
785  if(is_data_type_quantized_asymmetric(src->data_type()))
786  {
788  }
789  else
790  {
792  }
793  }
794 
795  if(dst->total_size() != 0)
796  {
797  const TensorShape output_shape = misc::shape_calculator::compute_depthwise_convolution_shape(*src, *weights, info);
800  }
801 
802  return Status{};
803 }
804 } // namespace
805 
807 {
808  ARM_COMPUTE_ERROR_ON_NULLPTR(src, weights, dst);
809  ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(src, weights, (biases != nullptr) ? biases : nullptr, dst, info));
810 
811  _conv_info = info.pad_stride_info;
812  _depth_multiplier = info.depth_multiplier;
813  _dilation = info.dilation;
814  _has_biases = (biases != nullptr);
815 
817  {
818  const auto input_scale = src->quantization_info().uniform().scale;
819  const auto output_scale = dst->quantization_info().uniform().scale;
820 
821  auto weights_scale = weights->quantization_info().scale();
823  {
824  for(size_t i = 1; i < weights->dimension(channel_idx); ++i)
825  {
826  weights_scale.push_back(weights_scale.front());
827  }
828  }
829 
830  for(const auto &s : weights_scale)
831  {
832  int32_t out_mult = 0;
833  int32_t out_shift = 0;
834  const float multiplier = input_scale * s / output_scale;
835  arm_compute::quantization::calculate_quantized_multiplier(multiplier, &out_mult, &out_shift);
836 
837  _output_multiplier.push_back(out_mult);
838  _output_shift.push_back(out_shift);
839  }
840  }
841 
842  switch(weights->data_type())
843  {
844  case DataType::QASYMM8:
845  _func = &CpuDepthwiseConv2dNativeKernel::run_depthwise<uint8_t, uint8_t>;
846  break;
848  _func = &CpuDepthwiseConv2dNativeKernel::run_depthwise<int8_t, int8_t>;
849  break;
851  if(src->data_type() == DataType::QASYMM8)
852  {
853  _func = &CpuDepthwiseConv2dNativeKernel::run_depthwise<uint8_t, int8_t>;
854  }
855  else
856  {
857  _func = &CpuDepthwiseConv2dNativeKernel::run_depthwise<int8_t, int8_t>;
858  }
859  break;
860 #ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
861  case DataType::F16:
862  _func = &CpuDepthwiseConv2dNativeKernel::run_depthwise<float16_t, float16_t>;
863  break;
864 #endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
865  case DataType::F32:
866  _func = &CpuDepthwiseConv2dNativeKernel::run_depthwise<float, float>;
867  break;
868  default:
869  ARM_COMPUTE_ERROR("Data type not supported");
870  break;
871  }
872 
874  auto_init_if_empty(*dst, src->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(output_shape).set_quantization_info(dst->quantization_info()));
875 
876  Window win = calculate_max_window(*dst, Steps());
877  ICpuKernel::configure(win);
878 }
879 
881 {
882  ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments(src, weights, biases, dst, info));
883  return Status{};
884 }
885 
886 template <typename T, typename TW, CpuDepthwiseConv2dNativeKernel::FloatEnalber<T>>
887 void CpuDepthwiseConv2dNativeKernel::run_depthwise(const ITensor *src, const ITensor *weights, const ITensor *biases,
888  ITensor *dst, const Window &window, bool has_biases)
889 {
892 
893  if(_depth_multiplier == 1)
894  {
895  depthwise_loop_multiplier1_fp<T>(src, weights, biases, dst, _conv_info, _dilation, window, has_biases);
896  }
897  else
898  {
899  depthwise_loop_generic_fp<T>(src, weights, biases, dst, _conv_info, _dilation, _depth_multiplier, window, has_biases);
900  }
901 }
902 
903 template <typename T, typename TW, CpuDepthwiseConv2dNativeKernel::Quantized8bitEnalber<T>>
904 void CpuDepthwiseConv2dNativeKernel::run_depthwise(const ITensor *src, const ITensor *weights, const ITensor *biases,
905  ITensor *dst, const Window &window, bool has_biases)
906 {
909 
910  if(_depth_multiplier == 1)
911  {
912  depthwise_loop_multiplier1_quantized<T, TW>(src, weights, biases, dst, _conv_info, _dilation, _output_multiplier, _output_shift, window, has_biases);
913  }
914  else
915  {
916  const bool is_pow2 = ((_depth_multiplier & (_depth_multiplier - 1)) == 0);
917  const bool is_quantized_per_tensor = !(is_data_type_quantized_per_channel(weights->info()->data_type()));
918 
919  if(is_pow2 && is_quantized_per_tensor && _depth_multiplier >= 8)
920  {
921  depthwise_loop_pow2_quantized_per_tensor<T, TW>(src, weights, biases, dst, _conv_info, _dilation, _depth_multiplier, _output_multiplier, _output_shift, window, has_biases);
922  }
923  else
924  {
925  depthwise_loop_generic_quantized<T, TW>(src, weights, biases, dst, _conv_info, _dilation, _depth_multiplier, _output_multiplier, _output_shift, window, has_biases);
926  }
927  }
928 }
929 
931 {
932  ARM_COMPUTE_UNUSED(info);
935  ARM_COMPUTE_ERROR_ON(_func == nullptr);
936 
937  const auto src = tensors.get_const_tensor(TensorType::ACL_SRC_0);
938  const auto weights = tensors.get_const_tensor(TensorType::ACL_SRC_1);
939  const auto biases = tensors.get_const_tensor(TensorType::ACL_SRC_2);
940  auto dst = tensors.get_tensor(TensorType::ACL_DST);
941  (this->*_func)(src, weights, biases, dst, window, _has_biases);
942 }
943 
945 {
946  return "CpuDepthwiseConv2dNativeKernel";
947 }
948 } // namespace kernels
949 } // namespace cpu
950 } // namespace arm_compute
bool is_data_type_quantized(DataType dt)
Check if a given data type is of quantized type.
Definition: Utils.h:981
const size_t weights_stride_z
Window calculate_max_window(const ValidRegion &valid_region, const Steps &steps, bool skip_border, BorderSize border_size)
SimpleTensor< float > w
Definition: DFT.cpp:156
Traits defined on Arm® Neon™ vectors.
const Window & window() const
The maximum window the kernel can be executed on.
Definition: IKernel.cpp:28
Shape of a tensor.
Definition: TensorShape.h:39
TensorShape compute_depthwise_convolution_shape(const ITensorInfo &input, const ITensorInfo &weights, const ConvolutionInfo &info)
Calculate the depthwise convolution output shape of a tensor.
const size_t weights_height
uint32x2_t vmovn(const uint64x2_t &a)
Definition: movn.h:39
#define ARM_COMPUTE_RETURN_ERROR_ON_CPU_F16_UNSUPPORTED(tensor)
Definition: Validate.h:115
const size_t conv_pad_left
virtual size_t dimension(size_t index) const =0
Return the size of the requested dimension.
const size_t input_stride_y
SimpleTensor< float > b
Definition: DFT.cpp:157
const size_t weights_stride_y
const size_t input_depth
#define ARM_COMPUTE_ERROR(msg)
Print the given message then throw an std::runtime_error.
Definition: Error.h:352
uint8x16_t vloadq(const uint8_t *ptr)
Definition: load.h:58
#define ARM_COMPUTE_RETURN_ON_ERROR(status)
Checks if a status contains an error and returns it.
Definition: Error.h:204
size_t element_size_from_data_type(DataType dt)
The size in bytes of the data type.
Definition: Utils.h:185
virtual DataType data_type() const =0
Data type used for each element of the tensor.
uint8x8_t vadd(const uint8x8_t &a, const uint8x8_t &b)
Definition: add.h:39
1 channel, 1 F32 per channel
#define ARM_COMPUTE_ERROR_ON(cond)
If the condition is true then an error message is printed and an exception thrown.
Definition: Error.h:466
const DataLayout data_layout
Definition: Im2Col.cpp:151
Store the tensor&#39;s metadata.
Definition: ITensorInfo.h:40
#define ARM_COMPUTE_ERROR_THROW_ON(status)
Definition: Error.h:455
uint8x8_t vsub(const uint8x8_t &a, const uint8x8_t &b)
Definition: sub.h:39
Status calculate_quantized_multiplier(float multiplier, int32_t *quant_multiplier, int32_t *shift, bool ignore_epsilon=false)
Calculate quantized representation of multiplier.
const size_t input_stride_z
Status class.
Definition: Error.h:52
#define ARM_COMPUTE_RETURN_ERROR_ON(cond)
If the condition is true, an error is returned.
Definition: Error.h:296
decltype(strategy::transforms) typedef type
Interface for CPU tensor.
Definition: ITensor.h:36
#define ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DIMENSIONS(...)
Definition: Validate.h:284
SimpleTensor< float > src
Definition: DFT.cpp:155
Copyright (c) 2017-2021 Arm Limited.
1 channel, 1 F16 per channel
const size_t num_read_elements_per_iteration
#define ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(...)
Definition: Validate.h:159
1 channel, 1 S32 per channel
uint32x2_t vqmovn(const uint64x2_t &a)
Definition: movn.h:52
unsigned int depth_multiplier
Multiplier to apply to input&#39;s depth to retrieve the output depth.
Definition: Types.h:1873
const DataType data_type
Definition: Im2Col.cpp:150
const ITensor * get_const_tensor(int id) const
Get constant tensor of a given id.
Definition: ITensorPack.cpp:54
const size_t input_width
static constexpr size_t DimX
Alias for dimension 0 also known as X dimension.
Definition: Window.h:43
#define ARM_COMPUTE_UNUSED(...)
To avoid unused variables warnings.
Definition: Error.h:152
library fill(src, distribution, 0)
bool is_data_type_quantized_per_channel(DataType dt)
Check if a given data type is of per channel type.
Definition: Utils.h:1058
PadStrideInfo pad_stride_info
Convolution info (Pads, strides,...)
Definition: Types.h:1872
quantized, asymmetric fixed-point 8-bit number unsigned
Class to describe a number of elements in each dimension.
Definition: Steps.h:40
const size_t input_max_offset
int16x4_t vreinterpret(const uint16x4_t &a)
Definition: reinterpret.h:44
uint8x8_t vmin(const uint8x8_t &a, const uint8x8_t &b)
Definition: min.h:39
Size2D dilation
Dilation, in elements, across x and y.
Definition: Types.h:1875
UniformQuantizationInfo uniform() const
Return per layer quantization info.
bool auto_init_if_empty(ITensorInfo &info, const TensorShape &shape, int num_channels, DataType data_type, QuantizationInfo quantization_info=QuantizationInfo())
Auto initialize the tensor info (shape, number of channels and data type) if the current assignment i...
virtual std::unique_ptr< T > clone() const =0
Provide a clone of the current object of class T.
virtual ITensorInfo * info() const =0
Interface to be implemented by the child class to return the tensor&#39;s metadata.
const std::vector< float > & scale() const
Scale vector accessor.
uint8x8_t vgetlow(const uint8x16_t val)
Definition: getlow.h:39
void end(TokenStream &in, bool &valid)
Definition: MLGOParser.cpp:290
uint8x16_t vcombine(const uint8x8_t &a, const uint8x8_t &b)
Definition: combine.h:39
static constexpr size_t DimW
Alias for dimension 3 also known as W dimension.
Definition: Window.h:49
const size_t weights_width
virtual QuantizationInfo quantization_info() const =0
Get the quantization settings (scale and offset) of the tensor.
uint16x8_t vmlal(const uint16x8_t &a, const uint8x8_t &b, const uint8x8_t &c)
Definition: mla.h:76
#define ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(k)
Definition: Validate.h:915
const size_t conv_stride_x
bool is_data_type_quantized_asymmetric(DataType dt)
Check if a given data type is of asymmetric quantized type.
Definition: Utils.h:1003
void configure(const ITensorInfo *src, const ITensorInfo *weights, const ITensorInfo *biases, ITensorInfo *dst, const ConvolutionInfo &info)
Initialize the function&#39;s source, destination and parameters.
quantized, symmetric per channel fixed-point 8-bit number
uint8x8_t vgethigh(const uint8x16_t val)
Definition: gethigh.h:39
void run_op(ITensorPack &tensors, const Window &window, const ThreadInfo &info) override
Execute the kernel on the passed window.
static constexpr size_t DimY
Alias for dimension 1 also known as Y dimension.
Definition: Window.h:45
ScaleKernelInfo info(interpolation_policy, default_border_mode, PixelValue(), sampling_policy, false)
ITensor * get_tensor(int id)
Get tensor of a given id from the pac.
Definition: ITensorPack.cpp:64
Information about executing thread and CPU.
Definition: CPPTypes.h:158
const size_t conv_pad_top
T fma(T x, T y, T z)
Computes (x*y) + z as if to infinite precision and rounded only once to fit the result type...
static constexpr size_t DimZ
Alias for dimension 2 also known as Z dimension.
Definition: Window.h:47
#define ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(...)
Definition: Validate.h:541
Num samples, height, width, channels.
const size_t input_height
#define ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(t, c,...)
Definition: Validate.h:788
const uint32_t x_end
uint8x8_t vload(const uint8_t *ptr)
Definition: load.h:39
void vstore(uint8_t *ptr, uint8x8_t val)
Definition: store.h:39
Tensor packing service.
Definition: ITensorPack.h:39
#define ARM_COMPUTE_ERROR_ON_NULLPTR(...)
Definition: Validate.h:157
uint8x8_t vdup_n(uint8_t value, traits::vector_64_tag)
Definition: dup_n.h:41
void execute_window_loop(const Window &w, L &&lambda_function, Ts &&... iterators)
Iterate through the passed window, automatically adjusting the iterators and calling the lambda_funct...
Definition: Helpers.inl:77
static Status validate(const ITensorInfo *src, const ITensorInfo *weights, const ITensorInfo *biases, const ITensorInfo *dst, const ConvolutionInfo &info)
Static function to check if given info will lead to a valid configuration.
quantized, asymmetric fixed-point 8-bit number signed
Includes all wrapper headers at once.
uint8x8_t vmla(const uint8x8_t &a, const uint8x8_t &b, const uint8x8_t &c)
Definition: mla.h:46
size_t get_data_layout_dimension_index(const DataLayout data_layout, const DataLayoutDimension data_layout_dimension)
Get the index of the given dimension.
Definition: Helpers.inl:193
const size_t conv_stride_y
const uint32_t x_start
uint16x8_t vmovl(const uint8x8_t &a)
Definition: movl.h:39
uint8x8_t vmax(const uint8x8_t &a, const uint8x8_t &b)
Definition: max.h:39
const uint32_t x_step
Describe a multidimensional execution window.
Definition: Window.h:39
#define ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(f, s)
Definition: Validate.h:201
const uint32_t x_leftover_start