Compute Library
 22.05
impl.cpp
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2018-2022 Arm Limited.
3  *
4  * SPDX-License-Identifier: MIT
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a copy
7  * of this software and associated documentation files (the "Software"), to
8  * deal in the Software without restriction, including without limitation the
9  * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10  * sell copies of the Software, and to permit persons to whom the Software is
11  * furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in all
14  * copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22  * SOFTWARE.
23  */
25 
26 namespace arm_compute
27 {
28 namespace cpu
29 {
30 template <typename T>
31 void fused_batch_normalization_dwc_nhwc(const ITensor *dwc_weights, const ITensor *dwc_bias, ITensor *fused_weights, ITensor *fused_bias,
32  const ITensor *bn_mean, const ITensor *bn_var, const ITensor *bn_beta, const ITensor *bn_gamma, float epsilon, const Window &window)
33 {
34  using ScalarType = T;
35  const int size = 16 / dwc_weights->info()->element_size();
37 
38  const bool run_in_place_weights = (fused_weights == nullptr) || (fused_weights == dwc_weights);
39  const bool run_in_place_bias = (fused_bias == nullptr) || (dwc_bias != nullptr && fused_bias == dwc_bias);
40 
41  // Set build options
42  Window win = window;
43  win.set(Window::DimX, Window::Dimension(0, 1, 1));
44 
45  const int window_step_x = size;
46  const auto window_start_x = static_cast<int>(window.x().start());
47  const auto window_end_x = static_cast<int>(window.x().end());
48 
49  Iterator dwc_w_in(dwc_weights, win);
50  Iterator dwc_w_out(run_in_place_weights ? dwc_weights : fused_weights, win);
51 
52  const auto dwc_bias_in = (dwc_bias != nullptr ? reinterpret_cast<ScalarType *>(dwc_bias->ptr_to_element(Coordinates(0, 0))) : nullptr);
53  auto dwc_bias_out = (run_in_place_bias ? dwc_bias_in : reinterpret_cast<ScalarType *>(fused_bias->ptr_to_element(Coordinates(0, 0))));
54 
55  const auto input_mean = reinterpret_cast<const ScalarType *>(bn_mean->ptr_to_element(Coordinates(0, 0)));
56  const auto input_var = reinterpret_cast<const ScalarType *>(bn_var->ptr_to_element(Coordinates(0, 0)));
57  const auto input_gamma = (bn_gamma != nullptr) ? reinterpret_cast<const ScalarType *>(bn_gamma->ptr_to_element(Coordinates(0, 0))) : nullptr;
58  const auto input_beta = (bn_beta != nullptr) ? reinterpret_cast<const ScalarType *>(bn_beta->ptr_to_element(Coordinates(0, 0))) : nullptr;
59 
60  auto mean_vec = wrapper::vdup_n(ScalarType(0), ExactTagType{});
61  auto var_vec = wrapper::vdup_n(ScalarType(0), ExactTagType{});
62  auto gamma_vec = wrapper::vdup_n(ScalarType(1), ExactTagType{});
63  auto beta_vec = wrapper::vdup_n(ScalarType(0), ExactTagType{});
64  auto rvar_vec = wrapper::vdup_n(ScalarType(0), ExactTagType{});
65  auto dwc_bias_vec = wrapper::vdup_n(ScalarType(0), ExactTagType{});
66  const auto epsilon_vec = wrapper::vdup_n(ScalarType(epsilon), ExactTagType{});
67 
68  auto gamma = ScalarType(1.0);
69  auto beta = ScalarType(0.0);
70  auto dwc_bias_in_scalar = ScalarType(0);
71 
72  execute_window_loop(win, [&](const Coordinates & id)
73  {
74  int x = window_start_x;
75  for(; x <= (window_end_x - window_step_x); x += window_step_x)
76  {
77  var_vec = wrapper::vloadq(input_var + x);
78  if(input_gamma != nullptr)
79  {
80  gamma_vec = wrapper::vloadq(input_gamma + x);
81  }
82 
83  if((id[2] == 0) && (id[1] == 0))
84  {
85  mean_vec = wrapper::vloadq(input_mean + x);
86 
87  // Construct vectors
88  if(input_beta != nullptr)
89  {
90  beta_vec = wrapper::vloadq(input_beta + x);
91  }
92 
93  if(dwc_bias_in != nullptr)
94  {
95  dwc_bias_vec = wrapper::vloadq(dwc_bias_in + x);
96  }
97 
98  auto dwc_bias_tmp_vec = wrapper::vmul(wrapper::vsub(dwc_bias_vec, mean_vec), wrapper::vinvsqrt(wrapper::vadd(var_vec, epsilon_vec)));
99  dwc_bias_tmp_vec = wrapper::vadd(wrapper::vmul(dwc_bias_tmp_vec, gamma_vec), beta_vec);
100  wrapper::vstore(dwc_bias_out + x, dwc_bias_tmp_vec);
101  }
102 
103  auto dwc_w_in_ptr = reinterpret_cast<const ScalarType *>(dwc_w_in.ptr());
104  auto dwc_w_out_ptr = reinterpret_cast<ScalarType *>(dwc_w_out.ptr());
105 
106  auto wn = wrapper::vloadq(dwc_w_in_ptr + x);
107  rvar_vec = wrapper::vinvsqrt(wrapper::vadd(var_vec, epsilon_vec));
108  wn = wrapper::vmul(wn, rvar_vec);
109  wn = wrapper::vmul(wn, gamma_vec);
110 
111  // Store results
112  wrapper::vstore(dwc_w_out_ptr + x, wn);
113  }
114 
115  // Compute left-over elements
116  for(; x < window_end_x; ++x)
117  {
118  auto var = input_var[x];
119  if(input_gamma != nullptr)
120  {
121  gamma = input_gamma[x];
122  }
123 
124  if(id[2] == 0 && id[1] == 0)
125  {
126  auto mean = input_mean[x];
127  if(input_beta != nullptr)
128  {
129  beta = input_beta[x];
130  }
131  if(dwc_bias_in != nullptr)
132  {
133  dwc_bias_in_scalar = dwc_bias_in[x];
134  }
135 
136  auto dwc_bias_tmp_scalar = (dwc_bias_in_scalar - mean) / std::sqrt(var + ScalarType(epsilon));
137  dwc_bias_out[x] = (dwc_bias_tmp_scalar * gamma) + beta;
138  }
139 
140  const auto dwc_w_in_ptr = reinterpret_cast<const ScalarType *>(dwc_w_in.ptr());
141  auto dwc_w_out_ptr = reinterpret_cast<ScalarType *>(dwc_w_out.ptr());
142 
143  *(dwc_w_out_ptr + x) = *(dwc_w_in_ptr + x) / std::sqrt(var + ScalarType(epsilon)) * gamma;
144  }
145  },
146  dwc_w_in, dwc_w_out);
147 }
148 
149 template void fused_batch_normalization_dwc_nhwc<float32_t>(const ITensor *dwc_weights, const ITensor *dwc_bias, ITensor *fused_weights, ITensor *fused_bias,
150  const ITensor *bn_mean, const ITensor *bn_var, const ITensor *bn_beta, const ITensor *bn_gamma, float epsilon, const Window &window);
151 
152 #if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(ENABLE_FP16_KERNELS)
153 template void fused_batch_normalization_dwc_nhwc<float16_t>(const ITensor *dwc_weights, const ITensor *dwc_bias, ITensor *fused_weights, ITensor *fused_bias,
154  const ITensor *bn_mean, const ITensor *bn_var, const ITensor *bn_beta, const ITensor *bn_gamma, float epsilon, const Window &window);
155 #endif /* defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(ENABLE_FP16_KERNELS) */
156 
157 } // namespace cpu
158 } // namespace arm_compute
uint8_t * ptr_to_element(const Coordinates &id) const
Return a pointer to the element at the passed coordinates.
Definition: ITensor.h:63
float32x2_t vinvsqrt(const float32x2_t &a)
Definition: invsqrt.h:47
uint8x16_t vloadq(const uint8_t *ptr)
Definition: load.h:58
uint8x8_t vadd(const uint8x8_t &a, const uint8x8_t &b)
Definition: add.h:39
uint8x8_t vsub(const uint8x8_t &a, const uint8x8_t &b)
Definition: sub.h:39
Describe one of the image&#39;s dimensions with a start, end and step.
Definition: Window.h:79
Interface for CPU tensor.
Definition: ITensor.h:36
Copyright (c) 2017-2022 Arm Limited.
typename neon_bitvector< T, BW >::tag_type neon_bitvector_tag_t
Helper type template to get the tag type of a neon vector.
Definition: traits.h:132
static constexpr size_t DimX
Alias for dimension 0 also known as X dimension.
Definition: Window.h:43
template void fused_batch_normalization_dwc_nhwc< float32_t >(const ITensor *dwc_weights, const ITensor *dwc_bias, ITensor *fused_weights, ITensor *fused_bias, const ITensor *bn_mean, const ITensor *bn_var, const ITensor *bn_beta, const ITensor *bn_gamma, float epsilon, const Window &window)
Coordinates of an item.
Definition: Coordinates.h:37
virtual ITensorInfo * info() const =0
Interface to be implemented by the child class to return the tensor&#39;s metadata.
constexpr uint8_t * ptr() const
Return a pointer to the current pixel.
Definition: Helpers.inl:139
virtual size_t element_size() const =0
Element size in bytes calculated as data_size() * num_channels()
void set(size_t dimension, const Dimension &dim)
Set the values of a given dimension.
Definition: Window.inl:49
uint8x8_t vmul(const uint8x8_t &a, const uint8x8_t &b)
Definition: mul.h:39
void fused_batch_normalization_dwc_nhwc(const ITensor *dwc_weights, const ITensor *dwc_bias, ITensor *fused_weights, ITensor *fused_bias, const ITensor *bn_mean, const ITensor *bn_var, const ITensor *bn_beta, const ITensor *bn_gamma, float epsilon, const Window &window)
Definition: impl.cpp:31
void vstore(uint8_t *ptr, uint8x8_t val)
Definition: store.h:39
uint8x8_t vdup_n(uint8_t value, traits::vector_64_tag)
Definition: dup_n.h:41
void execute_window_loop(const Window &w, L &&lambda_function, Ts &&... iterators)
Iterate through the passed window, automatically adjusting the iterators and calling the lambda_funct...
Definition: Helpers.inl:77
constexpr int end() const
Return the end of the dimension.
Definition: Window.h:101
Iterator updated by execute_window_loop for each window element.
Definition: Helpers.h:46
constexpr int start() const
Return the start of the dimension.
Definition: Window.h:96
Describe a multidimensional execution window.
Definition: Window.h:39
constexpr const Dimension & x() const
Alias to access the first dimension of the window.
Definition: Window.h:158