Compute Library
 21.08
NEMath.inl
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2016-2021 Arm Limited.
3  *
4  * SPDX-License-Identifier: MIT
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a copy
7  * of this software and associated documentation files (the "Software"), to
8  * deal in the Software without restriction, including without limitation the
9  * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10  * sell copies of the Software, and to permit persons to whom the Software is
11  * furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in all
14  * copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22  * SOFTWARE.
23  */
25 
26 #include <cmath>
27 #include <limits>
28 
29 namespace arm_compute
30 {
31 /** Exponent polynomial coefficients */
32 const std::array<float32x4_t, 8> exp_tab =
33 {
34  {
35  vdupq_n_f32(1.f),
36  vdupq_n_f32(0.0416598916054f),
37  vdupq_n_f32(0.500000596046f),
38  vdupq_n_f32(0.0014122662833f),
39  vdupq_n_f32(1.00000011921f),
40  vdupq_n_f32(0.00833693705499f),
41  vdupq_n_f32(0.166665703058f),
42  vdupq_n_f32(0.000195780929062f),
43  }
44 };
45 
46 /** Logarithm polynomial coefficients */
47 const std::array<float32x4_t, 8> log_tab =
48 {
49  {
50  vdupq_n_f32(-2.29561495781f),
51  vdupq_n_f32(-2.47071170807f),
52  vdupq_n_f32(-5.68692588806f),
53  vdupq_n_f32(-0.165253549814f),
54  vdupq_n_f32(5.17591238022f),
55  vdupq_n_f32(0.844007015228f),
56  vdupq_n_f32(4.58445882797f),
57  vdupq_n_f32(0.0141278216615f),
58  }
59 };
60 
61 /** Sin polynomial coefficients */
62 constexpr float te_sin_coeff2 = 0.166666666666f; // 1/(2*3)
63 constexpr float te_sin_coeff3 = 0.05f; // 1/(4*5)
64 constexpr float te_sin_coeff4 = 0.023809523810f; // 1/(6*7)
65 constexpr float te_sin_coeff5 = 0.013888888889f; // 1/(8*9)
66 
67 #ifndef DOXYGEN_SKIP_THIS
68 inline float32x4_t vfloorq_f32(float32x4_t val)
69 {
70  static const float32x4_t CONST_1 = vdupq_n_f32(1.f);
71 
72  const int32x4_t z = vcvtq_s32_f32(val);
73  const float32x4_t r = vcvtq_f32_s32(z);
74 
75  return vbslq_f32(vcgtq_f32(r, val), vsubq_f32(r, CONST_1), r);
76 }
77 
78 inline float32x4_t vroundq_rte_f32(float32x4_t val)
79 {
80 #ifdef __aarch64__
81  return vrndnq_f32(val);
82 #else // __aarch64__
83  static const float32x4_t CONST_HALF_FLOAT = vdupq_n_f32(0.5f);
84  static const float32x4_t CONST_1_FLOAT = vdupq_n_f32(1.f);
85  static const int32x4_t CONST_1_INT = vdupq_n_s32(1);
86  const float32x4_t floor_val = vfloorq_f32(val);
87  const float32x4_t diff = vsubq_f32(val, floor_val);
88 
89  /*
90  * Select the floor value when (diff<0.5 || (diff==0.5 && floor_val%2==0).
91  * This condition is checked by vorrq_u32(vcltq_f32(diff, CONST_HALF_FLOAT) ,vandq_u32(vceqq_f32(diff, CONST_HALF_FLOAT) , vmvnq_u32(vtstq_s32(vandq_s32(vcvtq_s32_f32(floor_val), CONST_1_INT),CONST_1_INT))))
92  */
93 
94  return vbslq_f32(vorrq_u32(vcltq_f32(diff, CONST_HALF_FLOAT), vandq_u32(vceqq_f32(diff, CONST_HALF_FLOAT), vmvnq_u32(vtstq_s32(vandq_s32(vcvtq_s32_f32(floor_val), CONST_1_INT), CONST_1_INT)))),
95  floor_val, vaddq_f32(floor_val, CONST_1_FLOAT));
96 #endif // __aarch64__
97 }
98 
99 inline float32x2_t vinvsqrt_f32(float32x2_t x)
100 {
101  float32x2_t sqrt_reciprocal = vrsqrte_f32(x);
102  sqrt_reciprocal = vmul_f32(vrsqrts_f32(vmul_f32(x, sqrt_reciprocal), sqrt_reciprocal), sqrt_reciprocal);
103  sqrt_reciprocal = vmul_f32(vrsqrts_f32(vmul_f32(x, sqrt_reciprocal), sqrt_reciprocal), sqrt_reciprocal);
104 
105  return sqrt_reciprocal;
106 }
107 
108 inline float32x4_t vinvsqrtq_f32(float32x4_t x)
109 {
110  float32x4_t sqrt_reciprocal = vrsqrteq_f32(x);
111  sqrt_reciprocal = vmulq_f32(vrsqrtsq_f32(vmulq_f32(x, sqrt_reciprocal), sqrt_reciprocal), sqrt_reciprocal);
112  sqrt_reciprocal = vmulq_f32(vrsqrtsq_f32(vmulq_f32(x, sqrt_reciprocal), sqrt_reciprocal), sqrt_reciprocal);
113 
114  return sqrt_reciprocal;
115 }
116 
117 inline float32x2_t vinv_f32(float32x2_t x)
118 {
119  float32x2_t recip = vrecpe_f32(x);
120  recip = vmul_f32(vrecps_f32(x, recip), recip);
121  recip = vmul_f32(vrecps_f32(x, recip), recip);
122  return recip;
123 }
124 
125 inline float32x4_t vinvq_f32(float32x4_t x)
126 {
127  float32x4_t recip = vrecpeq_f32(x);
128  recip = vmulq_f32(vrecpsq_f32(x, recip), recip);
129  recip = vmulq_f32(vrecpsq_f32(x, recip), recip);
130  return recip;
131 }
132 
133 inline float32x4_t vtaylor_polyq_f32(float32x4_t x, const std::array<float32x4_t, 8> &coeffs)
134 {
135  float32x4_t A = vmlaq_f32(coeffs[0], coeffs[4], x);
136  float32x4_t B = vmlaq_f32(coeffs[2], coeffs[6], x);
137  float32x4_t C = vmlaq_f32(coeffs[1], coeffs[5], x);
138  float32x4_t D = vmlaq_f32(coeffs[3], coeffs[7], x);
139  float32x4_t x2 = vmulq_f32(x, x);
140  float32x4_t x4 = vmulq_f32(x2, x2);
141  float32x4_t res = vmlaq_f32(vmlaq_f32(A, B, x2), vmlaq_f32(C, D, x2), x4);
142  return res;
143 }
144 
145 inline float32x4_t vexpq_f32(float32x4_t x)
146 {
147  static const float32x4_t CONST_LN2 = vdupq_n_f32(0.6931471805f); // ln(2)
148  static const float32x4_t CONST_INV_LN2 = vdupq_n_f32(1.4426950408f); // 1/ln(2)
149  static const float32x4_t CONST_INF = vdupq_n_f32(std::numeric_limits<float>::infinity());
150  static const float32x4_t CONST_MAX_INPUT = vdupq_n_f32(88.7f);
151  static const float32x4_t CONST_0 = vdupq_n_f32(0.f);
152  static const int32x4_t CONST_NEGATIVE_126 = vdupq_n_s32(-126);
153 
154  // Perform range reduction [-log(2),log(2)]
155  int32x4_t m = vcvtq_s32_f32(vmulq_f32(x, CONST_INV_LN2));
156  float32x4_t val = vmlsq_f32(x, vcvtq_f32_s32(m), CONST_LN2);
157 
158  // Polynomial Approximation
159  float32x4_t poly = vtaylor_polyq_f32(val, exp_tab);
160 
161  // Reconstruct
162  poly = vreinterpretq_f32_s32(vqaddq_s32(vreinterpretq_s32_f32(poly), vqshlq_n_s32(m, 23)));
163  poly = vbslq_f32(vcltq_s32(m, CONST_NEGATIVE_126), CONST_0, poly); // Handle underflow
164  poly = vbslq_f32(vcgtq_f32(x, CONST_MAX_INPUT), CONST_INF, poly); // Handle overflow
165 
166  return poly;
167 }
168 
169 inline float32x4_t vlogq_f32(float32x4_t x)
170 {
171  static const int32x4_t CONST_127 = vdupq_n_s32(127); // 127
172  static const float32x4_t CONST_LN2 = vdupq_n_f32(0.6931471805f); // ln(2)
173 
174  // Extract exponent
175  int32x4_t m = vsubq_s32(vreinterpretq_s32_u32(vshrq_n_u32(vreinterpretq_u32_f32(x), 23)), CONST_127);
176  float32x4_t val = vreinterpretq_f32_s32(vsubq_s32(vreinterpretq_s32_f32(x), vshlq_n_s32(m, 23)));
177 
178  // Polynomial Approximation
179  float32x4_t poly = vtaylor_polyq_f32(val, log_tab);
180 
181  // Reconstruct
182  poly = vmlaq_f32(poly, vcvtq_f32_s32(m), CONST_LN2);
183 
184  return poly;
185 }
186 
187 inline float32x4_t vtanhq_f32(float32x4_t val)
188 {
189  static const float32x4_t CONST_1 = vdupq_n_f32(1.f);
190  static const float32x4_t CONST_2 = vdupq_n_f32(2.f);
191  static const float32x4_t CONST_MIN_TANH = vdupq_n_f32(-10.f);
192  static const float32x4_t CONST_MAX_TANH = vdupq_n_f32(10.f);
193  static const float32x4_t CONST_THR = vdupq_n_f32(5.e-3);
194  static const float32x4_t CONST_1_3 = vdupq_n_f32(0.3333333f);
195 
196  float32x4_t x = vminq_f32(vmaxq_f32(val, CONST_MIN_TANH), CONST_MAX_TANH);
197  // x * (1 - x^2/3) if |x| < 5.e-3 or (exp2x - 1) / (exp2x + 1) otherwise
198  float32x4_t exp2x = vbslq_f32(vcgtq_f32(vabsq_f32(x), CONST_THR), vexpq_f32(vmulq_f32(CONST_2, x)), vmulq_f32(x, x));
199  float32x4_t num = vbslq_f32(vcgtq_f32(vabsq_f32(x), CONST_THR), vsubq_f32(exp2x, CONST_1), vmulq_f32(CONST_1_3, exp2x));
200  float32x4_t den = vbslq_f32(vcgtq_f32(vabsq_f32(x), CONST_THR), vaddq_f32(exp2x, CONST_1), vsubq_f32(CONST_1, num));
201  float32x4_t tanh = vbslq_f32(vcgtq_f32(vabsq_f32(x), CONST_THR), vmulq_f32(num, vinvq_f32(den)), vmulq_f32(x, den));
202  return tanh;
203 }
204 
205 inline float32x4_t vpowq_f32(float32x4_t val, float32x4_t n)
206 {
207  return vexpq_f32(vmulq_f32(n, vlogq_f32(val)));
208 }
209 
210 inline float32x4_t vsinq_f32(float32x4_t val)
211 {
212  const float32x4_t pi_v = vdupq_n_f32(M_PI);
213  const float32x4_t pio2_v = vdupq_n_f32(M_PI / 2);
214  const float32x4_t ipi_v = vdupq_n_f32(1 / M_PI);
215 
216  //Find positive or negative
217  const int32x4_t c_v = vabsq_s32(vcvtq_s32_f32(vmulq_f32(val, ipi_v)));
218  const uint32x4_t sign_v = vcleq_f32(val, vdupq_n_f32(0));
219  const uint32x4_t odd_v = vandq_u32(vreinterpretq_u32_s32(c_v), vdupq_n_u32(1));
220 
221  uint32x4_t neg_v = veorq_u32(odd_v, sign_v);
222 
223  //Modulus a - (n * int(a*(1/n)))
224  float32x4_t ma = vsubq_f32(vabsq_f32(val), vmulq_f32(pi_v, vcvtq_f32_s32(c_v)));
225  const uint32x4_t reb_v = vcgeq_f32(ma, pio2_v);
226 
227  //Rebase a between 0 and pi/2
228  ma = vbslq_f32(reb_v, vsubq_f32(pi_v, ma), ma);
229 
230  //Taylor series
231  const float32x4_t ma2 = vmulq_f32(ma, ma);
232 
233  //2nd elem: x^3 / 3!
234  float32x4_t elem = vmulq_f32(vmulq_f32(ma, ma2), vdupq_n_f32(te_sin_coeff2));
235  float32x4_t res = vsubq_f32(ma, elem);
236 
237  //3rd elem: x^5 / 5!
238  elem = vmulq_f32(vmulq_f32(elem, ma2), vdupq_n_f32(te_sin_coeff3));
239  res = vaddq_f32(res, elem);
240 
241  //4th elem: x^7 / 7!float32x2_t vsin_f32(float32x2_t val)
242  elem = vmulq_f32(vmulq_f32(elem, ma2), vdupq_n_f32(te_sin_coeff4));
243  res = vsubq_f32(res, elem);
244 
245  //5th elem: x^9 / 9!
246  elem = vmulq_f32(vmulq_f32(elem, ma2), vdupq_n_f32(te_sin_coeff5));
247  res = vaddq_f32(res, elem);
248 
249  //Change of sign
250  neg_v = vshlq_n_u32(neg_v, 31);
251  res = vreinterpretq_f32_u32(veorq_u32(vreinterpretq_u32_f32(res), neg_v));
252  return res;
253 }
254 
255 inline float32x2_t vsin_f32(float32x2_t val)
256 {
257  const float32x2_t pi_v = vdup_n_f32(M_PI);
258  const float32x2_t pio2_v = vdup_n_f32(M_PI / 2);
259  const float32x2_t ipi_v = vdup_n_f32(1 / M_PI);
260 
261  //Find positive or negative
262  const int32x2_t c_v = vabs_s32(vcvt_s32_f32(vmul_f32(val, ipi_v)));
263  const uint32x2_t sign_v = vcle_f32(val, vdup_n_f32(0));
264  const uint32x2_t odd_v = vand_u32(vreinterpret_u32_s32(c_v), vdup_n_u32(1));
265 
266  uint32x2_t neg_v = veor_u32(odd_v, sign_v);
267 
268  //Modulus a - (n * int(a*(1/n)))
269  float32x2_t ma = vsub_f32(vabs_f32(val), vmul_f32(pi_v, vcvt_f32_s32(c_v)));
270  const uint32x2_t reb_v = vcge_f32(ma, pio2_v);
271 
272  //Rebase a between 0 and pi/2
273  ma = vbsl_f32(reb_v, vsub_f32(pi_v, ma), ma);
274 
275  //Taylor series
276  const float32x2_t ma2 = vmul_f32(ma, ma);
277 
278  //2nd elem: x^3 / 3!
279  float32x2_t elem = vmul_f32(vmul_f32(ma, ma2), vdup_n_f32(te_sin_coeff2));
280  float32x2_t res = vsub_f32(ma, elem);
281 
282  //3rd elem: x^5 / 5!
283  elem = vmul_f32(vmul_f32(elem, ma2), vdup_n_f32(te_sin_coeff3));
284  res = vadd_f32(res, elem);
285 
286  //4th elem: x^7 / 7!float32x2_t vsin_f32(float32x2_t val)
287  elem = vmul_f32(vmul_f32(elem, ma2), vdup_n_f32(te_sin_coeff4));
288  res = vsub_f32(res, elem);
289 
290  //5th elem: x^9 / 9!
291  elem = vmul_f32(vmul_f32(elem, ma2), vdup_n_f32(te_sin_coeff5));
292  res = vadd_f32(res, elem);
293 
294  //Change of sign
295  neg_v = vshl_n_u32(neg_v, 31);
296  res = vreinterpret_f32_u32(veor_u32(vreinterpret_u32_f32(res), neg_v));
297  return res;
298 }
299 
300 #endif /* DOXYGEN_SKIP_THIS */
301 
302 inline int32x4_t rounding_divide_by_pow2(int32x4_t x, int32x4_t exponent)
303 {
304  const int32x4_t shift_vec = vnegq_s32(exponent);
305  const int32x4_t fixup = vshrq_n_s32(vandq_s32(x, shift_vec), 31);
306  const int32x4_t fixed_up_x = vqaddq_s32(x, fixup);
307  return vrshlq_s32(fixed_up_x, shift_vec);
308 }
309 
310 inline int32x4_t rounding_divide_by_pow2(int32x4_t x, int exponent)
311 {
312  const int32x4_t shift_vec = vdupq_n_s32(-exponent);
313  const int32x4_t fixup = vshrq_n_s32(vandq_s32(x, shift_vec), 31);
314  const int32x4_t fixed_up_x = vqaddq_s32(x, fixup);
315  return vrshlq_s32(fixed_up_x, shift_vec);
316 }
317 
318 inline int32_t rounding_divide_by_pow2(int32_t x, int exponent)
319 {
320  const int32_t mask = (1 << exponent) - 1;
321  const int32_t threshold = (mask >> 1) + (x < 0 ? 1 : 0);
322  return (x >> exponent) + ((x & mask) > threshold ? 1 : 0);
323 }
324 
325 inline float32x4x4_t convert_uint8x16_to_float32x4x4(const uint8x16_t &in)
326 {
327  float32x4x4_t out;
328 
329  const auto tmp1 = vmovl_u8(vget_low_u8(in));
330  out.val[0] = vcvtq_f32_u32(vmovl_u16(vget_low_u16(tmp1)));
331  out.val[1] = vcvtq_f32_u32(vmovl_u16(vget_high_u16(tmp1)));
332 
333  const auto tmp2 = vmovl_u8(vget_high_u8(in));
334  out.val[2] = vcvtq_f32_u32(vmovl_u16(vget_low_u16(tmp2)));
335  out.val[3] = vcvtq_f32_u32(vmovl_u16(vget_high_u16(tmp2)));
336  return out;
337 }
338 
339 inline float32x4x4_t convert_int8x16_to_float32x4x4(const int8x16_t &in)
340 {
341  float32x4x4_t out;
342 
343  const auto tmp1 = vmovl_s8(vget_low_s8(in));
344  out.val[0] = vcvtq_f32_s32(vmovl_s16(vget_low_s16(tmp1)));
345  out.val[1] = vcvtq_f32_s32(vmovl_s16(vget_high_s16(tmp1)));
346 
347  const auto tmp2 = vmovl_s8(vget_high_s8(in));
348  out.val[2] = vcvtq_f32_s32(vmovl_s16(vget_low_s16(tmp2)));
349  out.val[3] = vcvtq_f32_s32(vmovl_s16(vget_high_s16(tmp2)));
350  return out;
351 }
352 
353 template <>
354 inline float32x4x4_t convert_to_float32x4x4(const uint8x16_t &in)
355 {
357 }
358 
359 template <>
360 inline float32x4x4_t convert_to_float32x4x4(const int8x16_t &in)
361 {
363 }
364 
365 inline void convert_float32x4x3_to_uint8x8x3(const float32x4x3_t &in1, const float32x4x3_t &in2, uint8x8x3_t &out)
366 {
367  out.val[0] = vqmovn_u16(vcombine_u16(vqmovn_u32(vcvtq_u32_f32(in1.val[0])),
368  vqmovn_u32(vcvtq_u32_f32(in2.val[0]))));
369  out.val[1] = vqmovn_u16(vcombine_u16(vqmovn_u32(vcvtq_u32_f32(in1.val[1])),
370  vqmovn_u32(vcvtq_u32_f32(in2.val[1]))));
371  out.val[2] = vqmovn_u16(vcombine_u16(vqmovn_u32(vcvtq_u32_f32(in1.val[2])),
372  vqmovn_u32(vcvtq_u32_f32(in2.val[2]))));
373 }
374 
375 inline void convert_float32x4x4_to_uint8x16(const float32x4x4_t &in, uint8x16_t &out)
376 {
377  const auto low = vcombine_u16(vqmovn_u32(vcvtq_u32_f32(in.val[0])),
378  vqmovn_u32(vcvtq_u32_f32(in.val[1])));
379  const auto high = vcombine_u16(vqmovn_u32(vcvtq_u32_f32(in.val[2])),
380  vqmovn_u32(vcvtq_u32_f32(in.val[3])));
381  out = vcombine_u8(vqmovn_u16(low), vqmovn_u16(high));
382 }
383 
384 inline void convert_float32x4x4_to_int8x16(const float32x4x4_t &in, int8x16_t &out)
385 {
386  const auto low = vcombine_s16(vqmovn_s32(vcvtq_s32_f32(in.val[0])),
387  vqmovn_s32(vcvtq_s32_f32(in.val[1])));
388  const auto high = vcombine_s16(vqmovn_s32(vcvtq_s32_f32(in.val[2])),
389  vqmovn_s32(vcvtq_s32_f32(in.val[3])));
390  out = vcombine_s8(vqmovn_s16(low), vqmovn_s16(high));
391 }
392 
393 template <>
394 inline uint8x16_t convert_float_to_int<float32x4x4_t, uint8x16_t>(const float32x4x4_t &in)
395 {
396  uint8x16_t out;
398  return out;
399 }
400 
401 template <>
402 inline float32x4x4_t convert_int_to_float<float32x4x4_t, uint8x16_t>(const uint8x16_t &in)
403 {
405 }
406 
407 template <>
408 inline int8x16_t convert_float_to_int<float32x4x4_t, int8x16_t>(const float32x4x4_t &in)
409 {
410  int8x16_t out;
412  return out;
413 }
414 
415 template <>
416 inline float32x4x4_t convert_int_to_float<float32x4x4_t, int8x16_t>(const int8x16_t &in)
417 {
419 }
420 
421 #ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
422 /** Exponent polynomial coefficients */
423 /** Logarithm polynomial coefficients */
424 #ifndef DOXYGEN_SKIP_THIS
425 inline float16x8_t vfloorq_f16(float16x8_t val)
426 {
427  static const float16x8_t CONST_1 = vdupq_n_f16(1.f);
428 
429  const int16x8_t z = vcvtq_s16_f16(val);
430  const float16x8_t r = vcvtq_f16_s16(z);
431 
432  return vbslq_f16(vcgtq_f16(r, val), vsubq_f16(r, CONST_1), r);
433 }
434 
435 inline float16x8_t vroundq_rte_f16(float16x8_t val)
436 {
437  return vrndnq_f16(val);
438 }
439 
440 inline float16x4_t vinvsqrt_f16(float16x4_t x)
441 {
442  float16x4_t sqrt_reciprocal = vrsqrte_f16(x);
443  sqrt_reciprocal = vmul_f16(vrsqrts_f16(vmul_f16(x, sqrt_reciprocal), sqrt_reciprocal), sqrt_reciprocal);
444  sqrt_reciprocal = vmul_f16(vrsqrts_f16(vmul_f16(x, sqrt_reciprocal), sqrt_reciprocal), sqrt_reciprocal);
445  return sqrt_reciprocal;
446 }
447 
448 inline float16x8_t vinvsqrtq_f16(float16x8_t x)
449 {
450  float16x8_t sqrt_reciprocal = vrsqrteq_f16(x);
451  sqrt_reciprocal = vmulq_f16(vrsqrtsq_f16(vmulq_f16(x, sqrt_reciprocal), sqrt_reciprocal), sqrt_reciprocal);
452  sqrt_reciprocal = vmulq_f16(vrsqrtsq_f16(vmulq_f16(x, sqrt_reciprocal), sqrt_reciprocal), sqrt_reciprocal);
453  return sqrt_reciprocal;
454 }
455 
456 inline float16x4_t vinv_f16(float16x4_t x)
457 {
458  float16x4_t recip = vrecpe_f16(x);
459  recip = vmul_f16(vrecps_f16(x, recip), recip);
460  recip = vmul_f16(vrecps_f16(x, recip), recip);
461  return recip;
462 }
463 
464 inline float16x8_t vinvq_f16(float16x8_t x)
465 {
466  float16x8_t recip = vrecpeq_f16(x);
467  recip = vmulq_f16(vrecpsq_f16(x, recip), recip);
468  recip = vmulq_f16(vrecpsq_f16(x, recip), recip);
469  return recip;
470 }
471 
472 inline float16x8_t vtanhq_f16(float16x8_t val)
473 {
474  const float16x8_t CONST_1 = vdupq_n_f16(1.f);
475  const float16x8_t CONST_2 = vdupq_n_f16(2.f);
476  const float16x8_t CONST_MIN_TANH = vdupq_n_f16(-10.f);
477  const float16x8_t CONST_MAX_TANH = vdupq_n_f16(10.f);
478 
479  const float16x8_t x = vminq_f16(vmaxq_f16(val, CONST_MIN_TANH), CONST_MAX_TANH);
480  const float16x8_t exp2x = vexpq_f16(vmulq_f16(CONST_2, x));
481  const float16x8_t num = vsubq_f16(exp2x, CONST_1);
482  const float16x8_t den = vaddq_f16(exp2x, CONST_1);
483  const float16x8_t tanh = vmulq_f16(num, vinvq_f16(den));
484  return tanh;
485 }
486 
487 inline float16x8_t vtaylor_polyq_f16(float16x8_t x, const std::array<float16x8_t, 8> &coeffs)
488 {
489  const float16x8_t A = vaddq_f16(coeffs[0], vmulq_f16(coeffs[4], x));
490  const float16x8_t B = vaddq_f16(coeffs[2], vmulq_f16(coeffs[6], x));
491  const float16x8_t C = vaddq_f16(coeffs[1], vmulq_f16(coeffs[5], x));
492  const float16x8_t D = vaddq_f16(coeffs[3], vmulq_f16(coeffs[7], x));
493  const float16x8_t x2 = vmulq_f16(x, x);
494  const float16x8_t x4 = vmulq_f16(x2, x2);
495  const float16x8_t res = vaddq_f16(vaddq_f16(A, vmulq_f16(B, x2)), vmulq_f16(vaddq_f16(C, vmulq_f16(D, x2)), x4));
496  return res;
497 }
498 
499 inline float16x8_t vexpq_f16(float16x8_t x)
500 {
501  const float32x4_t x_high = vcvt_f32_f16(vget_high_f16(x));
502  const float32x4_t x_low = vcvt_f32_f16(vget_low_f16(x));
503 
504  const float16x8_t res = vcombine_f16(vcvt_f16_f32(vexpq_f32(x_low)), vcvt_f16_f32(vexpq_f32(x_high)));
505  return res;
506 }
507 
508 inline float16x8_t vlogq_f16(float16x8_t x)
509 {
510  const float32x4_t x_high = vcvt_f32_f16(vget_high_f16(x));
511  const float32x4_t x_low = vcvt_f32_f16(vget_low_f16(x));
512 
513  const float16x8_t res = vcombine_f16(vcvt_f16_f32(vlogq_f32(x_low)), vcvt_f16_f32(vlogq_f32(x_high)));
514  return res;
515 }
516 
517 inline float16x8_t vpowq_f16(float16x8_t val, float16x8_t n)
518 {
519  float32x4_t n0_f32 = vcvt_f32_f16(vget_low_f16(n));
520  float32x4_t n1_f32 = vcvt_f32_f16(vget_high_f16(n));
521  float32x4_t val0_f32 = vcvt_f32_f16(vget_low_f16(val));
522  float32x4_t val1_f32 = vcvt_f32_f16(vget_high_f16(val));
523 
524  float32x4_t res0_f32 = vexpq_f32(vmulq_f32(n0_f32, vlogq_f32(val0_f32)));
525  float32x4_t res1_f32 = vexpq_f32(vmulq_f32(n1_f32, vlogq_f32(val1_f32)));
526 
527  return vcombine_f16(vcvt_f16_f32(res0_f32), vcvt_f16_f32(res1_f32));
528 }
529 
530 inline float16x8_t vsinq_f16(float16x8_t val)
531 {
532  const float32x4_t val_high = vcvt_f32_f16(vget_high_f16(val));
533  const float32x4_t val_low = vcvt_f32_f16(vget_low_f16(val));
534 
535  const float32x4_t res_high = vsinq_f32(val_high);
536  const float32x4_t res_low = vsinq_f32(val_low);
537 
538  return vcombine_f16(vcvt_f16_f32(res_low), vcvt_f16_f32(res_high));
539 }
540 
541 inline float16x4_t vsin_f16(float16x4_t val)
542 {
543  const float32x4_t val_f32 = vcvt_f32_f16(val);
544  const float32x2_t val_high = vget_high_f32(val_f32);
545  const float32x2_t val_low = vget_low_f32(val_f32);
546 
547  const float32x2_t res_high = vsin_f32(val_high);
548  const float32x2_t res_low = vsin_f32(val_low);
549 
550  return vcvt_f16_f32(vcombine_f32(res_low, res_high));
551 }
552 
553 #endif /* DOXYGEN_SKIP_THIS */
554 #endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */
555 } // namespace arm_compute
float32x2_t vsin_f32(float32x2_t val)
Calculate sine.
float16x8_t vmaxq_f16(float16x8_t, float16x8_t)
Definition: clang-tidy.h:163
float32x4_t vtanhq_f32(float32x4_t val)
Calculate hyperbolic tangent.
uint16x8_t vcvtq_f16_s16(float16x8_t)
Definition: clang-tidy.h:118
float32x4_t vinvsqrtq_f32(float32x4_t x)
Calculate inverse square root.
constexpr float te_sin_coeff5
Definition: NEMath.inl:65
float32x2_t vinv_f32(float32x2_t x)
Calculate reciprocal.
float16x8_t vsubq_f16(float16x8_t, float16x8_t)
Definition: clang-tidy.h:73
float16x8_t vmulq_f16(float16x8_t, float16x8_t)
Definition: clang-tidy.h:78
int8x16_t convert_float_to_int< float32x4x4_t, int8x16_t >(const float32x4x4_t &in)
Definition: NEMath.inl:408
float32x4x4_t convert_int_to_float< float32x4x4_t, int8x16_t >(const int8x16_t &in)
Definition: NEMath.inl:416
float16x8_t vrsqrteq_f16(float16x8_t)
Definition: clang-tidy.h:133
float32x4x4_t convert_to_float32x4x4(const T &in)
Converts to float32x4x4_t from the specified templated 16 elements vectors.
float32x4x4_t convert_int8x16_to_float32x4x4(const int8x16_t &in)
Converts from int8x16 to float32x4x4_t.
Definition: NEMath.inl:339
float32x4x4_t convert_uint8x16_to_float32x4x4(const uint8x16_t &in)
Converts from uint8x16 to float32x4x4_t.
Definition: NEMath.inl:325
float16x8_t vaddq_f16(float16x8_t, float16x8_t)
Definition: clang-tidy.h:68
#define M_PI
float16x8_t vrsqrtsq_f16(float16x8_t, float16x8_t)
Definition: clang-tidy.h:8
float32x4_t vtaylor_polyq_f32(float32x4_t x, const std::array< float32x4_t, 8 > &coeffs)
Perform a 7th degree polynomial approximation using Estrin&#39;s method.
Copyright (c) 2017-2021 Arm Limited.
float32x4_t vfloorq_f32(float32x4_t val)
Calculate floor of a vector.
const std::array< float32x4_t, 8 > exp_tab
Exponent polynomial coefficients.
Definition: NEMath.inl:32
float16x8_t vrecpsq_f16(float16x8_t, float16x8_t)
Definition: clang-tidy.h:158
float16x4_t vrsqrte_f16(float16x4_t)
Definition: clang-tidy.h:128
float32x4_t vpowq_f32(float32x4_t val, float32x4_t n)
Calculate n power of a number.
float16x4_t vrecpe_f16(float16x4_t)
Definition: clang-tidy.h:143
float16x8_t vrecpeq_f16(float16x8_t)
Definition: clang-tidy.h:148
float16x4_t vrecps_f16(float16x4_t, float16x4_t)
Definition: clang-tidy.h:153
void convert_float32x4x4_to_int8x16(const float32x4x4_t &in, int8x16_t &out)
Converts from float32x4x4_t to just one int8x16_t.
Definition: NEMath.inl:384
uint8x16_t convert_float_to_int< float32x4x4_t, uint8x16_t >(const float32x4x4_t &in)
Definition: NEMath.inl:394
constexpr float te_sin_coeff3
Definition: NEMath.inl:63
float16x4_t vmul_f16(float16x4_t, float16x4_t)
Definition: clang-tidy.h:28
float16x8_t vbslq_f16(uint16x8_t, float16x8_t, float16x8_t)
Definition: clang-tidy.h:103
float32x4_t vlogq_f32(float32x4_t x)
Calculate logarithm.
constexpr float te_sin_coeff4
Definition: NEMath.inl:64
int32x4_t rounding_divide_by_pow2(int32x4_t x, int32x4_t exponent)
Round to the nearest division by a power-of-two using exponent.
Definition: NEMath.inl:302
float32x2_t vinvsqrt_f32(float32x2_t x)
Calculate inverse square root.
float32x4_t vsinq_f32(float32x4_t val)
Calculate sine.
float32x4_t vroundq_rte_f32(float32x4_t val)
Calculate round value of a vector to nearest with ties to even.
float16x4_t vrsqrts_f16(float16x4_t, float16x4_t)
Definition: clang-tidy.h:3
float32x4_t vexpq_f32(float32x4_t x)
Calculate exponential.
constexpr float te_sin_coeff2
Sin polynomial coefficients.
Definition: NEMath.inl:62
float32x4_t vinvq_f32(float32x4_t x)
Calculate reciprocal.
float32x4x4_t convert_int_to_float< float32x4x4_t, uint8x16_t >(const uint8x16_t &in)
Definition: NEMath.inl:402
int16x8_t vcvtq_s16_f16(float16x8_t)
Definition: clang-tidy.h:63
void convert_float32x4x4_to_uint8x16(const float32x4x4_t &in, uint8x16_t &out)
Converts from two float32x4x4_t to just one uint8x16_t.
Definition: NEMath.inl:375
float16x8_t vminq_f16(float16x8_t, float16x8_t)
Definition: clang-tidy.h:168
uint16x8_t vcgtq_f16(float16x8_t, float16x8_t)
Definition: clang-tidy.h:98
void convert_float32x4x3_to_uint8x8x3(const float32x4x3_t &in1, const float32x4x3_t &in2, uint8x8x3_t &out)
Converts from two float32x4x3_t to just one uint8x8x3_t.
Definition: NEMath.inl:365
const std::array< float32x4_t, 8 > log_tab
Logarithm polynomial coefficients.
Definition: NEMath.inl:47