25 #ifndef ACL_SRC_CORE_NEON_SVEMATH_INL
26 #define ACL_SRC_CORE_NEON_SVEMATH_INL
31 #if defined(__ARM_FEATURE_SVE) && defined(ARM_COMPUTE_ENABLE_SVE)
34 #define M_PI (3.14159265358979323846)
39 inline svfloat32_t svtaylor_poly_f32_z(svbool_t pg,
50 const auto A = svmla_f32_z(pg, coeff_1, coeff_5, x);
51 const auto B = svmla_f32_z(pg, coeff_3, coeff_7, x);
52 const auto C = svmla_f32_z(pg, coeff_2, coeff_6, x);
53 const auto D = svmla_f32_z(pg, coeff_4, coeff_8, x);
54 const auto x2 = svmul_f32_z(pg, x, x);
55 const auto x4 = svmul_f32_z(pg, x2, x2);
56 const auto res = svmla_f32_z(pg, svmla_f32_z(pg, A, B, x2), svmla_f32_z(pg, C, D, x2), x4);
60 inline svfloat16_t svtaylor_poly_f16_z(svbool_t pg,
71 const auto A = svmla_f16_z(pg, coeff_1, coeff_5, x);
72 const auto B = svmla_f16_z(pg, coeff_3, coeff_7, x);
73 const auto C = svmla_f16_z(pg, coeff_2, coeff_6, x);
74 const auto D = svmla_f16_z(pg, coeff_4, coeff_8, x);
75 const auto x2 = svmul_f16_z(pg, x, x);
76 const auto x4 = svmul_f16_z(pg, x2, x2);
77 const auto res = svmla_f16_z(pg, svmla_f16_z(pg, A, B, x2), svmla_f16_z(pg, C, D, x2), x4);
81 inline svfloat16_t svinv_f16_z(svbool_t pg, svfloat16_t x)
83 auto recip = svrecpe_f16(x);
84 recip = svmul_f16_z(pg, svrecps_f16(x, recip), recip);
85 recip = svmul_f16_z(pg, svrecps_f16(x, recip), recip);
89 inline svfloat32_t svinv_f32_z(svbool_t pg, svfloat32_t x)
91 auto recip = svrecpe_f32(x);
92 recip = svmul_f32_z(pg, svrecps_f32(x, recip), recip);
93 recip = svmul_f32_z(pg, svrecps_f32(x, recip), recip);
97 static const uint32_t svexp_f32_coeff[] = {
105 inline svfloat32_t svexp_f32_z(svbool_t pg, svfloat32_t x)
107 const auto c1 = svreinterpret_f32_u32(svdup_n_u32(svexp_f32_coeff[0]));
108 const auto c2 = svreinterpret_f32_u32(svdup_n_u32(svexp_f32_coeff[1]));
109 const auto c3 = svreinterpret_f32_u32(svdup_n_u32(svexp_f32_coeff[2]));
110 const auto c4 = svreinterpret_f32_u32(svdup_n_u32(svexp_f32_coeff[3]));
111 const auto c5 = svreinterpret_f32_u32(svdup_n_u32(svexp_f32_coeff[4]));
113 const auto shift = svreinterpret_f32_u32(svdup_n_u32(0x4b00007f));
114 const auto inv_ln2 = svreinterpret_f32_u32(svdup_n_u32(0x3fb8aa3b));
115 const auto neg_ln2_hi =
116 svreinterpret_f32_u32(svdup_n_u32(0xbf317200));
117 const auto neg_ln2_lo =
118 svreinterpret_f32_u32(svdup_n_u32(0xb5bfbe8e));
120 const auto inf = svdup_n_f32(std::numeric_limits<float>::infinity());
121 const auto max_input = svdup_n_f32(88.37f);
122 const auto zero = svdup_n_f32(0.f);
123 const auto min_input = svdup_n_f32(-86.64f);
139 const auto z = svmla_f32_z(pg, shift, x, inv_ln2);
140 const auto n = svsub_f32_z(pg, z, shift);
141 const auto scale = svreinterpret_f32_u32(svlsl_n_u32_z(pg, svreinterpret_u32_f32(z), 23));
145 const auto r_hi = svmla_f32_z(pg, x, n, neg_ln2_hi);
146 const auto r = svmla_f32_z(pg, r_hi, n, neg_ln2_lo);
150 const auto r2 = svmul_f32_z(pg, r, r);
152 const auto p1 = svmul_f32_z(pg, c1, r);
153 const auto p23 = svmla_f32_z(pg, c2, c3, r);
154 const auto p45 = svmla_f32_z(pg, c4, c5, r);
155 const auto p2345 = svmla_f32_z(pg, p23, p45, r2);
156 const auto p12345 = svmla_f32_z(pg, p1, p2345, r2);
158 auto poly = svmla_f32_z(pg,
scale, p12345,
scale);
161 poly = svsel_f32(svcmplt_f32(pg, x, min_input), zero, poly);
162 poly = svsel_f32(svcmpgt_f32(pg, x, max_input), inf, poly);
167 inline svfloat16_t svexp_f16_z(svbool_t pg, svfloat16_t x)
169 auto bottom = svcvt_f32_z(pg, x);
170 auto pg_top = svptrue_b16();
171 auto top = svcvt_f32_z(pg_top, svreinterpret_f16(svrevh_z(svptrue_b16(), svreinterpret_u32(x))));
173 bottom = svexp_f32_z(pg, bottom);
174 top = svexp_f32_z(pg_top, top);
176 return svtrn1(svcvt_f16_z(pg, bottom), svcvt_f16_z(pg_top, top));
179 #ifdef ARM_COMPUTE_ENABLE_SVE2
181 inline svfloat16_t svexp_f16_z_sve2(svbool_t pg, svfloat16_t x)
183 auto bottom = svcvt_f32_z(pg, x);
184 auto top = svcvtlt_f32_x(pg, x);
187 bottom = svexp_f32_z(pg, bottom);
188 top = svexp_f32_z(pg_top, top);
190 return svcvtnt_f16_m(svcvt_f16_z(pg, bottom), pg_top, top);
193 #endif // ARM_COMPUTE_ENABLE_SVE2
195 inline svfloat32_t svtanh_f32_z(svbool_t pg, svfloat32_t val)
197 const svfloat32_t CONST_1 = svdup_n_f32(1.f);
198 const svfloat32_t CONST_2 = svdup_n_f32(2.f);
199 const svfloat32_t CONST_MIN_TANH = svdup_n_f32(-10.f);
200 const svfloat32_t CONST_MAX_TANH = svdup_n_f32(10.f);
202 svfloat32_t x = svmin_f32_z(pg, svmax_f32_z(pg, val, CONST_MIN_TANH), CONST_MAX_TANH);
203 svfloat32_t exp2x = svexp_f32_z(pg, svmul_f32_z(pg, CONST_2, x));
204 svfloat32_t num = svsub_f32_z(pg, exp2x, CONST_1);
205 svfloat32_t den = svadd_f32_z(pg, exp2x, CONST_1);
206 svfloat32_t tanh = svdiv_f32_z(pg, num, den);
210 inline svfloat16_t svtanh_f16_z(svbool_t pg, svfloat16_t val)
212 const svfloat16_t CONST_1 = svdup_n_f16(1.f);
213 const svfloat16_t CONST_2 = svdup_n_f16(2.f);
214 const svfloat16_t CONST_MIN_TANH = svdup_n_f16(-10.f);
215 const svfloat16_t CONST_MAX_TANH = svdup_n_f16(10.f);
217 const svfloat16_t x = svmin_f16_z(pg, svmax_f16_z(pg, val, CONST_MIN_TANH), CONST_MAX_TANH);
218 const svfloat16_t exp2x = svexp_f16_z(pg, svmul_f16_z(pg, CONST_2, x));
219 const svfloat16_t num = svsub_f16_z(pg, exp2x, CONST_1);
220 const svfloat16_t den = svadd_f16_z(pg, exp2x, CONST_1);
221 const svfloat16_t tanh = svdiv_f16_z(pg, num, den);
225 inline svfloat32_t svlog_f32_z(svbool_t pg, svfloat32_t x)
228 const svfloat32_t log_tab_1 = svdup_n_f32(-2.29561495781f);
229 const svfloat32_t log_tab_2 = svdup_n_f32(-2.47071170807f);
230 const svfloat32_t log_tab_3 = svdup_n_f32(-5.68692588806f);
231 const svfloat32_t log_tab_4 = svdup_n_f32(-0.165253549814f);
232 const svfloat32_t log_tab_5 = svdup_n_f32(5.17591238022f);
233 const svfloat32_t log_tab_6 = svdup_n_f32(0.844007015228f);
234 const svfloat32_t log_tab_7 = svdup_n_f32(4.58445882797f);
235 const svfloat32_t log_tab_8 = svdup_n_f32(0.0141278216615f);
237 const auto CONST_127 = svdup_n_s32(127);
238 const auto CONST_LN2 = svdup_n_f32(0.6931471805f);
241 auto m = svsub_s32_z(pg, svasr_n_s32_z(pg, svreinterpret_s32_f32(x), 23), CONST_127);
242 auto val = svreinterpret_f32_s32(svsub_s32_z(pg, svreinterpret_s32_f32(x), svlsl_n_s32_z(pg, m, 23)));
245 auto poly = svtaylor_poly_f32_z(pg, val, log_tab_1, log_tab_2, log_tab_3, log_tab_4, log_tab_5, log_tab_6,
246 log_tab_7, log_tab_8);
249 poly = svmla_f32_z(pg, poly, svcvt_f32_s32_z(pg, m), CONST_LN2);
254 inline svfloat16_t svlog_f16_z(svbool_t pg, svfloat16_t x)
256 auto bottom = svcvt_f32_z(pg, x);
257 auto pg_top = svptrue_b16();
258 auto top = svcvt_f32_z(pg_top, svreinterpret_f16(svrevh_z(svptrue_b16(), svreinterpret_u32(x))));
260 bottom = svlog_f32_z(pg, bottom);
261 top = svlog_f32_z(pg_top, top);
263 return svtrn1(svcvt_f16_z(pg, bottom), svcvt_f16_z(pg_top, top));
266 #ifdef ARM_COMPUTE_ENABLE_SVE2
268 inline svfloat16_t svlog_f16_z_sve2(svbool_t pg, svfloat16_t x)
270 auto bottom = svcvt_f32_z(pg, x);
271 auto top = svcvtlt_f32_x(pg, x);
274 bottom = svlog_f32_z(pg, bottom);
275 top = svlog_f32_z(pg_top, top);
277 return svcvtnt_f16_m(svcvt_f16_z(pg, bottom), pg_top, top);
280 #endif // ARM_COMPUTE_ENABLE_SVE2
282 inline svfloat32_t svsin_f32_z(svbool_t pg, svfloat32_t val)
284 using ScalarType = float;
285 using IntType = uint32_t;
292 const auto pi_v = wrapper::svdup_n(ScalarType(
M_PI));
293 const auto pio2_v = wrapper::svdup_n(ScalarType(
M_PI / 2));
294 const auto ipi_v = wrapper::svdup_n(ScalarType(1 /
M_PI));
297 const auto c_v = svabs_z(pg, wrapper::svcvt_z<int32_t>(pg, svmul_z(pg, val, ipi_v)));
298 const auto sign_v = svcmple(pg, val, wrapper::svdup_n(ScalarType(0)));
299 const auto odd_v = svcmpne(pg, svand_z(pg, wrapper::svreinterpret<IntType>(c_v), wrapper::svdup_n(IntType(1))),
300 wrapper::svdup_n(IntType(0)));
302 auto neg_v = sveor_z(pg, odd_v, sign_v);
305 auto ma = svsub_z(pg, svabs_z(pg, val), svmul_z(pg, pi_v, wrapper::svcvt_z<ScalarType>(pg, c_v)));
306 const auto reb_v = svcmpge(pg, ma, pio2_v);
309 ma = svsel(reb_v, svsub_z(pg, pi_v, ma), ma);
312 const auto ma2 = svmul_z(pg, ma, ma);
315 auto elem = svmul_z(pg, svmul_z(pg, ma, ma2), wrapper::svdup_n(ScalarType(
te_sin_coeff2)));
316 auto res = svsub_z(pg, ma, elem);
319 elem = svmul_z(pg, svmul_z(pg, elem, ma2), wrapper::svdup_n(ScalarType(
te_sin_coeff3)));
320 res = svadd_z(pg, res, elem);
323 elem = svmul_z(pg, svmul_z(pg, elem, ma2), wrapper::svdup_n(ScalarType(
te_sin_coeff4)));
324 res = svsub_z(pg, res, elem);
327 elem = svmul_z(pg, svmul_z(pg, elem, ma2), wrapper::svdup_n(ScalarType(
te_sin_coeff5)));
328 res = svadd_z(pg, res, elem);
331 res = svneg_m(res, neg_v, res);
335 inline svfloat16_t svsin_f16_z(svbool_t pg, svfloat16_t val)
337 auto bottom = svcvt_f32_z(pg, val);
338 auto pg_top = svptrue_b16();
339 auto top = svcvt_f32_z(pg_top, svreinterpret_f16(svrevh_z(svptrue_b16(), svreinterpret_u32(val))));
341 bottom = svsin_f32_z(pg, bottom);
342 top = svsin_f32_z(pg_top, top);
344 return svtrn1(svcvt_f16_z(pg, bottom), svcvt_f16_z(pg_top, top));
347 #ifdef ARM_COMPUTE_ENABLE_SVE2
349 inline svfloat16_t svsin_f16_z_sve2(svbool_t pg, svfloat16_t val)
351 auto bottom = svcvt_f32_z(pg, val);
352 auto top = svcvtlt_f32_x(pg, val);
355 bottom = svsin_f32_z(pg, bottom);
356 top = svsin_f32_z(pg_top, top);
358 return svcvtnt_f16_m(svcvt_f16_z(pg, bottom), pg_top, top);
361 #endif // ARM_COMPUTE_ENABLE_SVE2
363 inline svfloat32_t svpow_f32_z(svbool_t pg, svfloat32_t a, svfloat32_t
b)
365 return svexp_f32_z(pg, svmul_z(pg,
b, svlog_f32_z(pg, a)));
368 inline svfloat16_t svpow_f16_z(svbool_t pg, svfloat16_t a, svfloat16_t
b)
370 auto a_bottom = svcvt_f32_z(pg, a);
371 auto b_bottom = svcvt_f32_z(pg,
b);
373 auto pg_top = svptrue_b16();
374 auto a_top = svcvt_f32_z(pg_top, svreinterpret_f16(svrevh_z(svptrue_b16(), svreinterpret_u32(a))));
375 auto b_top = svcvt_f32_z(pg_top, svreinterpret_f16(svrevh_z(svptrue_b16(), svreinterpret_u32(
b))));
377 auto res_bottom = svpow_f32_z(pg, a_bottom, b_bottom);
378 auto res_top = svpow_f32_z(pg_top, a_top, b_top);
380 return svtrn1(svcvt_f16_z(pg, res_bottom), svcvt_f16_z(pg_top, res_top));
383 #ifdef ARM_COMPUTE_ENABLE_SVE2
385 inline svfloat16_t svpow_f16_z_sve2(svbool_t pg, svfloat16_t a, svfloat16_t
b)
387 auto a_bottom = svcvt_f32_z(pg, a);
388 auto b_bottom = svcvt_f32_z(pg,
b);
391 auto a_top = svcvtlt_f32_x(pg, a);
392 auto b_top = svcvtlt_f32_x(pg,
b);
394 auto res_bottom = svpow_f32_z(pg, a_bottom, b_bottom);
395 auto res_top = svpow_f32_z(pg_top, a_top, b_top);
397 return svcvtnt_f16_m(svcvt_f16_z(pg, res_bottom), pg_top, res_top);
400 #endif // ARM_COMPUTE_ENABLE_SVE2
402 #if defined(ARM_COMPUTE_ENABLE_SVE2)
404 inline svuint8_t convert_float_to_int<svuint8_t>(
const svfloat32_t &in_0,
405 const svfloat32_t &in_1,
406 const svfloat32_t &in_2,
407 const svfloat32_t &in_3)
410 const auto all_true_pg = svptrue_b32();
411 auto tmp_0 = svcvt_u32_f32_z(all_true_pg, in_0);
412 auto tmp_1 = svcvt_u32_f32_z(all_true_pg, in_1);
413 auto tmp_2 = svcvt_u32_f32_z(all_true_pg, in_2);
414 auto tmp_3 = svcvt_u32_f32_z(all_true_pg, in_3);
416 auto tmp_16_0 = svqxtnt_u32(svqxtnb_u32(tmp_0), tmp_1);
417 auto tmp_16_1 = svqxtnt_u32(svqxtnb_u32(tmp_2), tmp_3);
419 auto tmp_16_uzp_0 = svuzp1(tmp_16_0, tmp_16_0);
420 auto tmp_16_uzp_1 = svuzp2(tmp_16_0, tmp_16_0);
421 auto tmp_16_uzp_2 = svuzp1(tmp_16_1, tmp_16_1);
422 auto tmp_16_uzp_3 = svuzp2(tmp_16_1, tmp_16_1);
424 auto pg = svwhilelt_b16_s32(0, svcnth() / 2);
426 tmp_16_0 = svsplice(pg, tmp_16_uzp_0, tmp_16_uzp_1);
427 tmp_16_1 = svsplice(pg, tmp_16_uzp_2, tmp_16_uzp_3);
429 out = svqxtnt_u16(svqxtnb_u16(tmp_16_0), tmp_16_1);
431 auto out_uzp_0 = svuzp1(out, out);
432 auto out_uzp_1 = svuzp2(out, out);
434 pg = svwhilelt_b8_s32(0, svcntb() / 2);
435 out = svsplice(pg, out_uzp_0, out_uzp_1);
441 inline svint8_t convert_float_to_int<svint8_t>(
const svfloat32_t &in_0,
442 const svfloat32_t &in_1,
443 const svfloat32_t &in_2,
444 const svfloat32_t &in_3)
447 const auto all_true_pg = svptrue_b32();
448 auto tmp_0 = svcvt_s32_f32_z(all_true_pg, in_0);
449 auto tmp_1 = svcvt_s32_f32_z(all_true_pg, in_1);
450 auto tmp_2 = svcvt_s32_f32_z(all_true_pg, in_2);
451 auto tmp_3 = svcvt_s32_f32_z(all_true_pg, in_3);
453 auto tmp_16_0 = svqxtnt_s32(svqxtnb_s32(tmp_0), tmp_1);
454 auto tmp_16_1 = svqxtnt_s32(svqxtnb_s32(tmp_2), tmp_3);
456 auto tmp_16_uzp_0 = svuzp1(tmp_16_0, tmp_16_0);
457 auto tmp_16_uzp_1 = svuzp2(tmp_16_0, tmp_16_0);
458 auto tmp_16_uzp_2 = svuzp1(tmp_16_1, tmp_16_1);
459 auto tmp_16_uzp_3 = svuzp2(tmp_16_1, tmp_16_1);
461 auto pg = svwhilelt_b16_s32(0, svcnth() / 2);
463 tmp_16_0 = svsplice(pg, tmp_16_uzp_0, tmp_16_uzp_1);
464 tmp_16_1 = svsplice(pg, tmp_16_uzp_2, tmp_16_uzp_3);
466 out = svqxtnt_s16(svqxtnb_s16(tmp_16_0), tmp_16_1);
468 auto out_uzp_0 = svuzp1(out, out);
469 auto out_uzp_1 = svuzp2(out, out);
471 pg = svwhilelt_b8_s32(0, svcntb() / 2);
472 out = svsplice(pg, out_uzp_0, out_uzp_1);
481 #endif // ACL_SRC_CORE_NEON_SVEMATH_INL