Compute Library
 22.05
impl.cpp
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2018-2022 Arm Limited.
3  *
4  * SPDX-License-Identifier: MIT
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a copy
7  * of this software and associated documentation files (the "Software"), to
8  * deal in the Software without restriction, including without limitation the
9  * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10  * sell copies of the Software, and to permit persons to whom the Software is
11  * furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in all
14  * copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22  * SOFTWARE.
23  */
25 
26 namespace arm_compute
27 {
28 namespace cpu
29 {
30 template <typename T>
31 void fused_batch_normalization_conv(const ITensor *conv_weights, const ITensor *conv_bias, ITensor *fused_weights, ITensor *fused_bias,
32  const ITensor *bn_mean, const ITensor *bn_var, const ITensor *bn_beta, const ITensor *bn_gamma, float epsilon, const Window &window)
33 {
34  using ScalarType = T;
35  const int size = 16 / conv_weights->info()->element_size();
37 
38  const bool run_in_place_weights = (fused_weights == nullptr) || (fused_weights == conv_weights);
39  const bool run_in_place_bias = (fused_bias == nullptr) || (conv_bias != nullptr && fused_bias == conv_bias);
40 
41  // Set build options
42  Window win = window;
43  win.set(Window::DimX, Window::Dimension(0, 1, 1));
44 
45  const int window_step_x = size;
46  const auto window_start_x = static_cast<int>(window.x().start());
47  const auto window_end_x = static_cast<int>(window.x().end());
48 
49  Iterator conv_w_in(conv_weights, win);
50  Iterator conv_w_out(run_in_place_weights ? conv_weights : fused_weights, win);
51 
52  const auto conv_bias_in = (conv_bias != nullptr ? reinterpret_cast<ScalarType *>(conv_bias->ptr_to_element(Coordinates(0, 0))) : nullptr);
53  auto conv_bias_out = (run_in_place_bias ? conv_bias_in : reinterpret_cast<ScalarType *>(fused_bias->ptr_to_element(Coordinates(0, 0))));
54 
55  const auto input_mean = reinterpret_cast<const ScalarType *>(bn_mean->ptr_to_element(Coordinates(0, 0)));
56  const auto input_var = reinterpret_cast<const ScalarType *>(bn_var->ptr_to_element(Coordinates(0, 0)));
57  const auto input_gamma = (bn_gamma != nullptr) ? reinterpret_cast<const ScalarType *>(bn_gamma->ptr_to_element(Coordinates(0, 0))) : nullptr;
58  const auto input_beta = (bn_beta != nullptr) ? reinterpret_cast<const ScalarType *>(bn_beta->ptr_to_element(Coordinates(0, 0))) : nullptr;
59 
60  auto mean_vec = wrapper::vdup_n(ScalarType(0), ExactTagType{});
61  auto var_vec = wrapper::vdup_n(ScalarType(0), ExactTagType{});
62  auto gamma_vec = wrapper::vdup_n(ScalarType(1), ExactTagType{});
63  auto beta_vec = wrapper::vdup_n(ScalarType(0), ExactTagType{});
64  auto rvar_vec = wrapper::vdup_n(ScalarType(0), ExactTagType{});
65  const auto epsilon_vec = wrapper::vdup_n(ScalarType(epsilon), ExactTagType{});
66 
67  auto mean = ScalarType(0.0);
68  auto var = ScalarType(0.0);
69  auto gamma = ScalarType(1.0);
70  auto beta = ScalarType(0.0);
71  auto conv_bias_in_scalar = ScalarType(0.0);
72  execute_window_loop(win, [&](const Coordinates & id)
73  {
74  var = input_var[id[3]];
75  if(input_gamma != nullptr)
76  {
77  gamma = input_gamma[id[3]];
78  }
79 
80  if((id[0] == 0) && (id[1] == 0) && (id[2] == 0))
81  {
82  if(input_beta != nullptr)
83  {
84  beta = input_beta[id[3]];
85  beta_vec = wrapper::vdup_n(beta, ExactTagType{});
86  }
87 
88  // Construct vectors
89  mean = input_mean[id[3]];
90  mean_vec = wrapper::vdup_n(mean, ExactTagType{});
91 
92  if(conv_bias_in != nullptr)
93  {
94  conv_bias_in_scalar = conv_bias_in[id[3]];
95  }
96  auto conv_bias_tmp_scalar = (conv_bias_in_scalar - mean) / std::sqrt(var + ScalarType(epsilon));
97  conv_bias_out[id[3]] = (conv_bias_tmp_scalar * gamma) + beta;
98  }
99 
100  int x = window_start_x;
101  auto conv_w_in_ptr = reinterpret_cast<const ScalarType *>(conv_w_in.ptr());
102  auto conv_w_out_ptr = reinterpret_cast<ScalarType *>(conv_w_out.ptr());
103  var_vec = wrapper::vdup_n(var, ExactTagType{});
104  gamma_vec = wrapper::vdup_n(gamma, ExactTagType{});
105  rvar_vec = wrapper::vinvsqrt(wrapper::vadd(var_vec, epsilon_vec));
106 
107  for(; x <= (window_end_x - window_step_x); x += window_step_x)
108  {
109  auto wn = wrapper::vloadq(conv_w_in_ptr + x);
110  wn = wrapper::vmul(wn, rvar_vec);
111  wn = wrapper::vmul(wn, gamma_vec);
112 
113  // Store results
114  wrapper::vstore(conv_w_out_ptr + x, wn);
115  }
116 
117  // Compute left-over elements
118  for(; x < window_end_x; ++x)
119  {
120  *(conv_w_out_ptr + x) = *(conv_w_in_ptr + x) / std::sqrt(var + ScalarType(epsilon)) * gamma;
121  }
122  },
123  conv_w_in, conv_w_out);
124 }
125 
126 template void fused_batch_normalization_conv<float32_t>(const ITensor *conv_weights, const ITensor *conv_bias, ITensor *fused_weights, ITensor *fused_bias,
127  const ITensor *bn_mean, const ITensor *bn_var, const ITensor *bn_beta, const ITensor *bn_gamma, float epsilon, const Window &window);
128 
129 #if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(ENABLE_FP16_KERNELS)
130 template void fused_batch_normalization_conv<float16_t>(const ITensor *conv_weights, const ITensor *conv_bias, ITensor *fused_weights, ITensor *fused_bias,
131  const ITensor *bn_mean, const ITensor *bn_var, const ITensor *bn_beta, const ITensor *bn_gamma, float epsilon, const Window &window);
132 #endif /* defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(ENABLE_FP16_KERNELS) */
133 
134 } // namespace cpu
135 } // namespace arm_compute
uint8_t * ptr_to_element(const Coordinates &id) const
Return a pointer to the element at the passed coordinates.
Definition: ITensor.h:63
float32x2_t vinvsqrt(const float32x2_t &a)
Definition: invsqrt.h:47
uint8x16_t vloadq(const uint8_t *ptr)
Definition: load.h:58
uint8x8_t vadd(const uint8x8_t &a, const uint8x8_t &b)
Definition: add.h:39
Describe one of the image&#39;s dimensions with a start, end and step.
Definition: Window.h:79
Interface for CPU tensor.
Definition: ITensor.h:36
Copyright (c) 2017-2022 Arm Limited.
void fused_batch_normalization_conv(const ITensor *conv_weights, const ITensor *conv_bias, ITensor *fused_weights, ITensor *fused_bias, const ITensor *bn_mean, const ITensor *bn_var, const ITensor *bn_beta, const ITensor *bn_gamma, float epsilon, const Window &window)
Definition: impl.cpp:31
typename neon_bitvector< T, BW >::tag_type neon_bitvector_tag_t
Helper type template to get the tag type of a neon vector.
Definition: traits.h:132
static constexpr size_t DimX
Alias for dimension 0 also known as X dimension.
Definition: Window.h:43
Coordinates of an item.
Definition: Coordinates.h:37
virtual ITensorInfo * info() const =0
Interface to be implemented by the child class to return the tensor&#39;s metadata.
constexpr uint8_t * ptr() const
Return a pointer to the current pixel.
Definition: Helpers.inl:139
virtual size_t element_size() const =0
Element size in bytes calculated as data_size() * num_channels()
void set(size_t dimension, const Dimension &dim)
Set the values of a given dimension.
Definition: Window.inl:49
template void fused_batch_normalization_conv< float32_t >(const ITensor *conv_weights, const ITensor *conv_bias, ITensor *fused_weights, ITensor *fused_bias, const ITensor *bn_mean, const ITensor *bn_var, const ITensor *bn_beta, const ITensor *bn_gamma, float epsilon, const Window &window)
uint8x8_t vmul(const uint8x8_t &a, const uint8x8_t &b)
Definition: mul.h:39
void vstore(uint8_t *ptr, uint8x8_t val)
Definition: store.h:39
uint8x8_t vdup_n(uint8_t value, traits::vector_64_tag)
Definition: dup_n.h:41
void execute_window_loop(const Window &w, L &&lambda_function, Ts &&... iterators)
Iterate through the passed window, automatically adjusting the iterators and calling the lambda_funct...
Definition: Helpers.inl:77
constexpr int end() const
Return the end of the dimension.
Definition: Window.h:101
Iterator updated by execute_window_loop for each window element.
Definition: Helpers.h:46
constexpr int start() const
Return the start of the dimension.
Definition: Window.h:96
Describe a multidimensional execution window.
Definition: Window.h:39
constexpr const Dimension & x() const
Alias to access the first dimension of the window.
Definition: Window.h:158