Compute Library
 23.11
impl.cpp
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2021-2022 Arm Limited.
3  *
4  * SPDX-License-Identifier: MIT
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a copy
7  * of this software and associated documentation files (the "Software"), to
8  * deal in the Software without restriction, including without limitation the
9  * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10  * sell copies of the Software, and to permit persons to whom the Software is
11  * furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in all
14  * copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22  * SOFTWARE.
23  */
24 
25 #include "src/cpu/kernels/elementwise_binary/generic/sve/impl.h"
26 
27 #include "src/core/NEON/SVEMath.h"
28 
29 #include <arm_sve.h>
30 
31 namespace arm_compute
32 {
33 namespace cpu
34 {
35 using namespace arm_compute::wrapper;
36 
37 template <typename ScalarType>
38 void elementwise_arithmetic_op(
39  const ITensor *in1, const ITensor *in2, ITensor *out, ArithmeticOperation op, const Window &window)
40 {
41  using VectorType = typename sve_vector<ScalarType>::type;
42 
43  const auto all_true_pg = svptrue<ScalarType>();
44 
45  // Create input windows
46  Window input1_win = window.broadcast_if_dimension_le_one(in1->info()->tensor_shape());
47  Window input2_win = window.broadcast_if_dimension_le_one(in2->info()->tensor_shape());
48 
49  // Clear X Dimension on execution window as we handle manually
50  Window win = window;
51  win.set(Window::DimX, Window::Dimension(0, 1, 1));
52 
53  const auto window_start_x = static_cast<int>(window.x().start());
54  const auto window_end_x = static_cast<int>(window.x().end());
55  const bool is_broadcast_across_x = in1->info()->tensor_shape().x() != in2->info()->tensor_shape().x();
56 
57  if (is_broadcast_across_x)
58  {
59  const bool is_broadcast_input_2 = input2_win.x().step() == 0;
60  Window broadcast_win = is_broadcast_input_2 ? input2_win : input1_win;
61  Window non_broadcast_win = !is_broadcast_input_2 ? input2_win : input1_win;
62  const ITensor *broadcast_tensor = is_broadcast_input_2 ? in2 : in1;
63  const ITensor *non_broadcast_tensor = !is_broadcast_input_2 ? in2 : in1;
64 
65  // Clear X Dimension on execution window as we handle manually
66  non_broadcast_win.set(Window::DimX, Window::Dimension(0, 1, 1));
67 
68  Iterator broadcast_input(broadcast_tensor, broadcast_win);
69  Iterator non_broadcast_input(non_broadcast_tensor, non_broadcast_win);
70  Iterator output(out, win);
71 
72  execute_window_loop(
73  win,
74  [&](const Coordinates &)
75  {
76  auto output_ptr = reinterpret_cast<ScalarType *>(output.ptr());
77  const auto non_broadcast_input_ptr = reinterpret_cast<const ScalarType *>(non_broadcast_input.ptr());
78  const ScalarType broadcast_value = *reinterpret_cast<const ScalarType *>(broadcast_input.ptr());
79  const auto broadcast_vector = svdup_n(broadcast_value);
80 
81  int x = window_start_x;
82 
83  svbool_t pg = svwhilelt<ScalarType>(x, window_end_x);
84  do
85  {
86  const auto non_broadcast_vector = svld1(pg, non_broadcast_input_ptr + x);
87  VectorType res{};
88 
89  if (is_broadcast_input_2)
90  {
91  res = elementwise_arithmetic_op<typename sve_vector<ScalarType>::type>(pg, non_broadcast_vector,
92  broadcast_vector, op);
93  }
94  else
95  {
96  res = elementwise_arithmetic_op<typename sve_vector<ScalarType>::type>(
97  pg, broadcast_vector, non_broadcast_vector, op);
98  }
99  svst1(pg, output_ptr + x, res);
100 
101  x += svcnt<ScalarType>();
102  pg = svwhilelt<ScalarType>(x, window_end_x);
103  } while (svptest_any(all_true_pg, pg));
104  },
105  broadcast_input, non_broadcast_input, output);
106  }
107  else
108  {
109  // Clear X Dimension on execution window as we handle manually
110  input1_win.set(Window::DimX, Window::Dimension(0, 1, 1));
111  input2_win.set(Window::DimX, Window::Dimension(0, 1, 1));
112 
113  Iterator input1(in1, input1_win);
114  Iterator input2(in2, input2_win);
115  Iterator output(out, win);
116 
117  execute_window_loop(
118  win,
119  [&](const Coordinates &)
120  {
121  auto output_ptr = reinterpret_cast<ScalarType *>(output.ptr());
122  const auto input1_ptr = reinterpret_cast<const ScalarType *>(input1.ptr());
123  const auto input2_ptr = reinterpret_cast<const ScalarType *>(input2.ptr());
124 
125  int x = window_start_x;
126 
127  svbool_t pg = svwhilelt<ScalarType>(x, window_end_x);
128  do
129  {
130  const auto in1 = svld1(pg, input1_ptr + x);
131  const auto in2 = svld1(pg, input2_ptr + x);
132  const auto res = elementwise_arithmetic_op<typename sve_vector<ScalarType>::type>(pg, in1, in2, op);
133  svst1(pg, output_ptr + x, res);
134 
135  x += svcnt<ScalarType>();
136  pg = svwhilelt<ScalarType>(x, window_end_x);
137  } while (svptest_any(all_true_pg, pg));
138  },
139  input1, input2, output);
140  }
141 }
142 template void elementwise_arithmetic_op<float32_t>(
143  const ITensor *in1, const ITensor *in2, ITensor *out, const ArithmeticOperation op, const Window &window);
144 template void elementwise_arithmetic_op<float16_t>(
145  const ITensor *in1, const ITensor *in2, ITensor *out, const ArithmeticOperation op, const Window &window);
146 template void elementwise_arithmetic_op<int16_t>(
147  const ITensor *in1, const ITensor *in2, ITensor *out, const ArithmeticOperation op, const Window &window);
148 template void elementwise_arithmetic_op<int32_t>(
149  const ITensor *in1, const ITensor *in2, ITensor *out, const ArithmeticOperation op, const Window &window);
150 
151 template <typename InputScalarType, typename OutputScalarType>
152 void elementwise_comparison_op(
153  const ITensor *in1, const ITensor *in2, ITensor *out, ComparisonOperation op, const Window &window)
154 {
155  static_assert(sizeof(InputScalarType) >= sizeof(OutputScalarType),
156  "input data type's width should be equal to or greater than output data type's width");
157 
158  using OutputVectorType = typename sve_vector<OutputScalarType>::type;
159  const auto all_true_pg = svptrue<InputScalarType>();
160 
161  // Create input windows
162  Window input1_win = window.broadcast_if_dimension_le_one(in1->info()->tensor_shape());
163  Window input2_win = window.broadcast_if_dimension_le_one(in2->info()->tensor_shape());
164 
165  // Clear X Dimension on execution window as we handle manually
166  Window win = window;
167  win.set(Window::DimX, Window::Dimension(0, 1, 1));
168 
169  const auto window_start_x = static_cast<int>(window.x().start());
170  const auto window_end_x = static_cast<int>(window.x().end());
171  const bool is_broadcast_across_x = in1->info()->tensor_shape().x() != in2->info()->tensor_shape().x();
172 
173  if (is_broadcast_across_x)
174  {
175  const bool is_broadcast_input_2 = input2_win.x().step() == 0;
176  Window broadcast_win = is_broadcast_input_2 ? input2_win : input1_win;
177  Window non_broadcast_win = !is_broadcast_input_2 ? input2_win : input1_win;
178  const ITensor *broadcast_tensor = is_broadcast_input_2 ? in2 : in1;
179  const ITensor *non_broadcast_tensor = !is_broadcast_input_2 ? in2 : in1;
180 
181  // Clear X Dimension on execution window as we handle manually
182  non_broadcast_win.set(Window::DimX, Window::Dimension(0, 1, 1));
183 
184  Iterator broadcast_input(broadcast_tensor, broadcast_win);
185  Iterator non_broadcast_input(non_broadcast_tensor, non_broadcast_win);
186  Iterator output(out, win);
187 
188  execute_window_loop(
189  win,
190  [&](const Coordinates &)
191  {
192  auto output_ptr = reinterpret_cast<OutputScalarType *>(output.ptr());
193  const auto non_broadcast_input_ptr =
194  reinterpret_cast<const InputScalarType *>(non_broadcast_input.ptr());
195  const InputScalarType broadcast_value =
196  *reinterpret_cast<const InputScalarType *>(broadcast_input.ptr());
197  const auto broadcast_vector = svdup_n(broadcast_value);
198 
199  int x = window_start_x;
200 
201  svbool_t pg = svwhilelt<InputScalarType>(x, window_end_x);
202  do
203  {
204  const auto non_broadcast_vector = svld1(pg, non_broadcast_input_ptr + x);
205  const svbool_t output_pg = narrow_to_byte_predicate<sizeof(InputScalarType)>(pg);
206  OutputVectorType res{};
207  if (is_broadcast_input_2)
208  {
209  res = elementwise_comparison_op<typename sve_vector<InputScalarType>::type,
210  typename sve_vector<OutputScalarType>::type>(
211  pg, non_broadcast_vector, broadcast_vector, op);
212  }
213  else
214  {
215  res = elementwise_comparison_op<typename sve_vector<InputScalarType>::type,
216  typename sve_vector<OutputScalarType>::type>(
217  pg, broadcast_vector, non_broadcast_vector, op);
218  }
219  svst1(output_pg, output_ptr + x, res);
220 
221  x += svcnt<InputScalarType>();
222  pg = svwhilelt<InputScalarType>(x, window_end_x);
223  } while (svptest_any(all_true_pg, pg));
224  },
225  broadcast_input, non_broadcast_input, output);
226  }
227  else
228  {
229  // Clear X Dimension on execution window as we handle manually
230  input1_win.set(Window::DimX, Window::Dimension(0, 1, 1));
231  input2_win.set(Window::DimX, Window::Dimension(0, 1, 1));
232 
233  Iterator input1(in1, input1_win);
234  Iterator input2(in2, input2_win);
235  Iterator output(out, win);
236 
237  execute_window_loop(
238  win,
239  [&](const Coordinates &)
240  {
241  auto output_ptr = reinterpret_cast<OutputScalarType *>(output.ptr());
242  const auto input1_ptr = reinterpret_cast<const InputScalarType *>(input1.ptr());
243  const auto input2_ptr = reinterpret_cast<const InputScalarType *>(input2.ptr());
244 
245  int x = window_start_x;
246 
247  svbool_t pg = svwhilelt<InputScalarType>(x, window_end_x);
248  do
249  {
250  const auto in1 = svld1(pg, input1_ptr + x);
251  const auto in2 = svld1(pg, input2_ptr + x);
252  const auto res =
253  elementwise_comparison_op<typename sve_vector<InputScalarType>::type,
254  typename sve_vector<OutputScalarType>::type>(pg, in1, in2, op);
255  const svbool_t output_pg = narrow_to_byte_predicate<sizeof(InputScalarType)>(pg);
256  svst1(output_pg, output_ptr + x, res);
257 
258  x += svcnt<InputScalarType>();
259  pg = svwhilelt<InputScalarType>(x, window_end_x);
260  } while (svptest_any(all_true_pg, pg));
261  },
262  input1, input2, output);
263  }
264 }
265 
266 template void elementwise_comparison_op<float32_t>(
267  const ITensor *in1, const ITensor *in2, ITensor *out, const ComparisonOperation op, const Window &window);
268 template void elementwise_comparison_op<float16_t>(
269  const ITensor *in1, const ITensor *in2, ITensor *out, const ComparisonOperation op, const Window &window);
270 template void elementwise_comparison_op<uint8_t>(
271  const ITensor *in1, const ITensor *in2, ITensor *out, const ComparisonOperation op, const Window &window);
272 template void elementwise_comparison_op<int16_t>(
273  const ITensor *in1, const ITensor *in2, ITensor *out, const ComparisonOperation op, const Window &window);
274 template void elementwise_comparison_op<int32_t>(
275  const ITensor *in1, const ITensor *in2, ITensor *out, const ComparisonOperation op, const Window &window);
276 
277 template <>
278 svint32_t elementwise_pow<svint32_t>(svbool_t &pg, const svint32_t &a, const svint32_t &b)
279 {
280  return svcvt_s32_z(pg, svpow_z(pg, svcvt_f32_z(pg, a), svcvt_f32_z(pg, b)));
281 }
282 
283 template <>
284 svint32_t elementwise_div<svint32_t>(svbool_t &pg, const svint32_t &a, const svint32_t &b)
285 {
286  return svcvt_s32_z(pg, svdiv_z(pg, svcvt_f32_z(pg, a), svcvt_f32_z(pg, b)));
287 }
288 
289 template <>
290 svint16_t elementwise_div<svint16_t>(svbool_t &pg, const svint16_t &a, const svint16_t &b)
291 {
292  ARM_COMPUTE_UNUSED(pg, a, b);
293  ARM_COMPUTE_ERROR("Not supported");
294 }
295 
296 } // namespace cpu
297 } // namespace arm_compute
arm_compute::cpu::elementwise_arithmetic_op< int32_t >
template void elementwise_arithmetic_op< int32_t >(const ITensor *in1, const ITensor *in2, ITensor *out, const ArithmeticOperation op, const Window &window)
arm_compute::Window::Dimension::start
constexpr int start() const
Return the start of the dimension.
Definition: Window.h:96
type
decltype(strategy::transforms) typedef type
Definition: gemm_interleaved.hpp:261
arm_compute::ITensorInfo::tensor_shape
virtual const TensorShape & tensor_shape() const =0
Size for each dimension of the tensor.
arm_compute::cpu::elementwise_comparison_op< int32_t >
template void elementwise_comparison_op< int32_t >(const ITensor *in1, const ITensor *in2, ITensor *out, const ComparisonOperation op, const Window &window)
arm_compute::Window::Dimension::step
constexpr int step() const
Return the step of the dimension.
Definition: Window.h:106
arm_compute::Window::DimX
static constexpr size_t DimX
Alias for dimension 0 also known as X dimension.
Definition: Window.h:43
ARM_COMPUTE_ERROR
#define ARM_COMPUTE_ERROR(msg)
Print the given message then throw an std::runtime_error.
Definition: Error.h:354
arm_compute::cpu::elementwise_comparison_op< int16_t >
template void elementwise_comparison_op< int16_t >(const ITensor *in1, const ITensor *in2, ITensor *out, const ComparisonOperation op, const Window &window)
arm_compute::ITensor
Interface for CPU tensor.
Definition: ITensor.h:36
SVEMath.h
arm_compute::wrapper
Definition: abs.h:31
arm_compute::cpu::elementwise_comparison_op< uint8_t >
template void elementwise_comparison_op< uint8_t >(const ITensor *in1, const ITensor *in2, ITensor *out, const ComparisonOperation op, const Window &window)
arm_compute::ITensor::info
virtual ITensorInfo * info() const =0
Interface to be implemented by the child class to return the tensor's metadata.
arm_compute::Iterator::ptr
constexpr uint8_t * ptr() const
Return a pointer to the current pixel.
Definition: Helpers.inl:147
arm_compute::cpu::elementwise_arithmetic_op< float32_t >
template void elementwise_arithmetic_op< float32_t >(const ITensor *in1, const ITensor *in2, ITensor *out, const ArithmeticOperation op, const Window &window)
arm_compute::ComparisonOperation
ComparisonOperation
Supported comparison operations.
Definition: Types.h:132
arm_compute::execute_window_loop
void execute_window_loop(const Window &w, L &&lambda_function, Ts &&...iterators)
Iterate through the passed window, automatically adjusting the iterators and calling the lambda_funct...
Definition: Helpers.inl:74
arm_compute::Iterator
Iterator updated by execute_window_loop for each window element.
Definition: Helpers.h:46
arm_compute::cpu::elementwise_arithmetic_op< int16_t >
template void elementwise_arithmetic_op< int16_t >(const ITensor *in1, const ITensor *in2, ITensor *out, const ArithmeticOperation op, const Window &window)
arm_compute::Dimensions::x
T x() const
Alias to access the size of the first dimension.
Definition: Dimensions.h:86
arm_compute::cpu::elementwise_div< svint16_t >
svint16_t elementwise_div< svint16_t >(svbool_t &pg, const svint16_t &a, const svint16_t &b)
Definition: impl.cpp:290
arm_compute::cpu::elementwise_pow< svint32_t >
svint32_t elementwise_pow< svint32_t >(svbool_t &pg, const svint32_t &a, const svint32_t &b)
Definition: impl.cpp:278
ARM_COMPUTE_UNUSED
#define ARM_COMPUTE_UNUSED(...)
To avoid unused variables warnings.
Definition: Error.h:151
arm_compute::cpu::elementwise_comparison_op< float16_t >
template void elementwise_comparison_op< float16_t >(const ITensor *in1, const ITensor *in2, ITensor *out, const ComparisonOperation op, const Window &window)
arm_compute::Coordinates
Coordinates of an item.
Definition: Coordinates.h:37
arm_compute::Window::Dimension
Describe one of the image's dimensions with a start, end and step.
Definition: Window.h:79
arm_compute::Window::set
void set(size_t dimension, const Dimension &dim)
Set the values of a given dimension.
Definition: Window.inl:53
arm_compute::cpu::elementwise_arithmetic_op< float16_t >
template void elementwise_arithmetic_op< float16_t >(const ITensor *in1, const ITensor *in2, ITensor *out, const ArithmeticOperation op, const Window &window)
arm_compute::cpu::elementwise_div< svint32_t >
svint32_t elementwise_div< svint32_t >(svbool_t &pg, const svint32_t &a, const svint32_t &b)
Definition: impl.cpp:284
arm_compute::Window
Describe a multidimensional execution window.
Definition: Window.h:39
arm_compute::cpu::elementwise_arithmetic_op
void elementwise_arithmetic_op(const ITensor *in1, const ITensor *in2, ITensor *out, ArithmeticOperation op, const Window &window)
Definition: impl.cpp:38
arm_compute::test::validation::b
SimpleTensor< float > b
Definition: DFT.cpp:157
arm_compute
Copyright (c) 2017-2023 Arm Limited.
Definition: introduction.dox:24
impl.h
arm_compute::cpu::elementwise_comparison_op
void elementwise_comparison_op(const ITensor *in1, const ITensor *in2, ITensor *out, ComparisonOperation op, const Window &window)
Definition: impl.cpp:152
arm_compute::Window::broadcast_if_dimension_le_one
Window broadcast_if_dimension_le_one(const TensorShape &shape) const
Don't advance in the dimension where shape is less equal to 1.
Definition: Window.inl:134
arm_compute::Window::Dimension::end
constexpr int end() const
Return the end of the dimension.
Definition: Window.h:101
arm_compute::Window::x
constexpr const Dimension & x() const
Alias to access the first dimension of the window.
Definition: Window.h:158
arm_compute::cpu::elementwise_comparison_op< float32_t >
template void elementwise_comparison_op< float32_t >(const ITensor *in1, const ITensor *in2, ITensor *out, const ComparisonOperation op, const Window &window)
arm_compute::ArithmeticOperation
ArithmeticOperation
Available element-wise operations.
Definition: Types.h:421