Compute Library
 21.02
NEAsymm.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2017-2020 Arm Limited.
3  *
4  * SPDX-License-Identifier: MIT
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a copy
7  * of this software and associated documentation files (the "Software"), to
8  * deal in the Software without restriction, including without limitation the
9  * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10  * sell copies of the Software, and to permit persons to whom the Software is
11  * furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in all
14  * copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22  * SOFTWARE.
23  */
24 #ifndef ARM_COMPUTE_NEASYMM_H
25 #define ARM_COMPUTE_NEASYMM_H
26 
27 #include "src/core/NEON/NEMath.h"
29 #include <arm_neon.h>
30 
31 namespace arm_compute
32 {
33 using qasymm8x8_t = uint8x8_t; /**< 8 bit quantized asymmetric vector with 8 elements */
34 using qasymm8x8x2_t = uint8x8x2_t; /**< 8 bit quantized asymmetric vector with 16 elements */
35 using qasymm8x8x3_t = uint8x8x3_t; /**< 8 bit quantized asymmetric vector with 24 elements */
36 using qasymm8x8x4_t = uint8x8x4_t; /**< 8 bit quantized asymmetric vector with 32 elements */
37 using qasymm8x16_t = uint8x16_t; /**< 8 bit quantized asymmetric vector with 16 elements */
38 
39 using qasymm8x8_signed_t = int8x8_t; /**< 8 bit quantized signed asymmetric vector with 8 elements */
40 using qasymm8x8x2_signed_t = int8x8x2_t; /**< 8 bit quantized signed asymmetric vector with 16 elements */
41 using qasymm8x8x3_signed_t = int8x8x3_t; /**< 8 bit quantized signed asymmetric vector with 24 elements */
42 using qasymm8x8x4_signed_t = int8x8x4_t; /**< 8 bit quantized signed asymmetric vector with 32 elements */
43 using qasymm8x16_signed_t = int8x16_t; /**< 8 bit quantized signed asymmetric vector with 16 elements */
44 
45 /** Perform a multiply-accumulate on all 16 components of a QASYMM8 vector
46  *
47  * vd*vs + vo
48  *
49  * @param[in] vd Input vector value in QASYMM8 format
50  * @param[in] vs Vector multiplier in F32 format. The multiplier value must be duplicated across all four lanes.
51  * @param[in] vo Vector addend in F32 format. The addend value must be duplicated across all four lanes.
52  *
53  * @return A 16-component vector in QASYMM8 format, saturated to fit
54  */
55 uint8x16_t vmlaq_qasymm8(qasymm8x16_t vd, float32x4_t vs, float32x4_t vo);
56 
57 /** Perform a multiply-accumulate on all 16 components of a QASYMM8_SIGNED vector
58  *
59  * vd*vs + vo
60  *
61  * @param[in] vd Input vector value in QASYMM8_SIGNED format
62  * @param[in] vs Vector multiplier in F32 format. The multiplier value must be duplicated across all four lanes.
63  * @param[in] vo Vector addend in F32 format. The addend value must be duplicated across all four lanes.
64  *
65  * @return A 16-component vector in QASYMM8_SIGNED format, saturated to fit
66  */
67 int8x16_t vmlaq_qasymm8_signed(qasymm8x16_signed_t vd, float32x4_t vs, float32x4_t vo);
68 
69 /** Performs final quantization step on 16 elements
70  *
71  * @param[in] in_s32 Input to be quantized.
72  * @param[in] result_fixedpoint_multiplier Result multiplier parameter
73  * @param[in] result_shift Result shift parameter
74  * @param[in] result_offset_after_shift_s32 Result offset parameter
75  * @param[in] min_u8 Relu lower bound
76  * @param[in] max_u8 Relu upper bound
77  * @param[in] is_bounded_relu Specified if a fused bounded relu should be applied
78  *
79  * @return Quantized values
80  */
81 inline uint8x16_t finalize_quantization(int32x4x4_t &in_s32,
82  int result_fixedpoint_multiplier,
83  int32_t result_shift,
84  int32x4_t result_offset_after_shift_s32,
85  uint8x16_t min_u8,
86  uint8x16_t max_u8,
87  bool is_bounded_relu)
88 {
89  const static int32x4_t zero_s32 = vdupq_n_s32(0);
90 
91  if(result_shift < 0)
92  {
93  in_s32.val[0] = vmulq_n_s32(in_s32.val[0], (1 << (-result_shift)));
94  in_s32.val[1] = vmulq_n_s32(in_s32.val[1], (1 << (-result_shift)));
95  in_s32.val[2] = vmulq_n_s32(in_s32.val[2], (1 << (-result_shift)));
96  in_s32.val[3] = vmulq_n_s32(in_s32.val[3], (1 << (-result_shift)));
97 
98  in_s32.val[0] = vqrdmulhq_n_s32(in_s32.val[0], result_fixedpoint_multiplier);
99  in_s32.val[1] = vqrdmulhq_n_s32(in_s32.val[1], result_fixedpoint_multiplier);
100  in_s32.val[2] = vqrdmulhq_n_s32(in_s32.val[2], result_fixedpoint_multiplier);
101  in_s32.val[3] = vqrdmulhq_n_s32(in_s32.val[3], result_fixedpoint_multiplier);
102  }
103  else
104  {
105  // Fixed point multiplication with vector saturating rounding doubling multiply high with scalar
106  in_s32.val[0] = vqrdmulhq_n_s32(in_s32.val[0], result_fixedpoint_multiplier);
107  in_s32.val[1] = vqrdmulhq_n_s32(in_s32.val[1], result_fixedpoint_multiplier);
108  in_s32.val[2] = vqrdmulhq_n_s32(in_s32.val[2], result_fixedpoint_multiplier);
109  in_s32.val[3] = vqrdmulhq_n_s32(in_s32.val[3], result_fixedpoint_multiplier);
110 
111  // Round to the nearest division by a power-of-two using result_shift_s32
112  in_s32.val[0] = rounding_divide_by_pow2(in_s32.val[0], result_shift);
113  in_s32.val[1] = rounding_divide_by_pow2(in_s32.val[1], result_shift);
114  in_s32.val[2] = rounding_divide_by_pow2(in_s32.val[2], result_shift);
115  in_s32.val[3] = rounding_divide_by_pow2(in_s32.val[3], result_shift);
116  }
117 
118  // Add the offset terms
119  in_s32.val[0] = vaddq_s32(in_s32.val[0], result_offset_after_shift_s32);
120  in_s32.val[1] = vaddq_s32(in_s32.val[1], result_offset_after_shift_s32);
121  in_s32.val[2] = vaddq_s32(in_s32.val[2], result_offset_after_shift_s32);
122  in_s32.val[3] = vaddq_s32(in_s32.val[3], result_offset_after_shift_s32);
123 
124  // Saturate negative values
125  in_s32.val[0] = vmaxq_s32(in_s32.val[0], zero_s32);
126  in_s32.val[1] = vmaxq_s32(in_s32.val[1], zero_s32);
127  in_s32.val[2] = vmaxq_s32(in_s32.val[2], zero_s32);
128  in_s32.val[3] = vmaxq_s32(in_s32.val[3], zero_s32);
129 
130  // Convert S32 to S16
131  const int16x8x2_t in_s16 =
132  {
133  {
134  vcombine_s16(vqmovn_s32(in_s32.val[0]), vqmovn_s32(in_s32.val[1])),
135  vcombine_s16(vqmovn_s32(in_s32.val[2]), vqmovn_s32(in_s32.val[3]))
136  }
137  };
138 
139  // Convert S16 to U8
140  uint8x16_t out_u8 = vcombine_u8(vqmovun_s16(in_s16.val[0]), vqmovun_s16(in_s16.val[1]));
141 
142  if(is_bounded_relu)
143  {
144  out_u8 = vmaxq_u8(out_u8, min_u8);
145  out_u8 = vminq_u8(out_u8, max_u8);
146  }
147 
148  return out_u8;
149 }
150 
151 /** Performs final quantization step on 16 elements
152  *
153  * @param[in] in_s32 Input to be quantized.
154  * @param[in] result_fixedpoint_multiplier Result multiplier parameter
155  * @param[in] result_shift Result shift parameter
156  * @param[in] result_offset_after_shift_s32 Result offset parameter
157  * @param[in] min_s8 Relu lower bound
158  * @param[in] max_s8 Relu upper bound
159  * @param[in] is_bounded_relu Specified if a fused bounded relu should be applied
160  *
161  * @return Quantized values
162  */
163 inline int8x16_t finalize_quantization(int32x4x4_t &in_s32,
164  int result_fixedpoint_multiplier,
165  int32_t result_shift,
166  int32x4_t result_offset_after_shift_s32,
167  int8x16_t min_s8,
168  int8x16_t max_s8,
169  bool is_bounded_relu)
170 {
171  if(result_shift < 0)
172  {
173  in_s32.val[0] = vmulq_n_s32(in_s32.val[0], (1 << (-result_shift)));
174  in_s32.val[1] = vmulq_n_s32(in_s32.val[1], (1 << (-result_shift)));
175  in_s32.val[2] = vmulq_n_s32(in_s32.val[2], (1 << (-result_shift)));
176  in_s32.val[3] = vmulq_n_s32(in_s32.val[3], (1 << (-result_shift)));
177 
178  in_s32.val[0] = vqrdmulhq_n_s32(in_s32.val[0], result_fixedpoint_multiplier);
179  in_s32.val[1] = vqrdmulhq_n_s32(in_s32.val[1], result_fixedpoint_multiplier);
180  in_s32.val[2] = vqrdmulhq_n_s32(in_s32.val[2], result_fixedpoint_multiplier);
181  in_s32.val[3] = vqrdmulhq_n_s32(in_s32.val[3], result_fixedpoint_multiplier);
182  }
183  else
184  {
185  // Fixed point multiplication with vector saturating rounding doubling multiply high with scalar
186  in_s32.val[0] = vqrdmulhq_n_s32(in_s32.val[0], result_fixedpoint_multiplier);
187  in_s32.val[1] = vqrdmulhq_n_s32(in_s32.val[1], result_fixedpoint_multiplier);
188  in_s32.val[2] = vqrdmulhq_n_s32(in_s32.val[2], result_fixedpoint_multiplier);
189  in_s32.val[3] = vqrdmulhq_n_s32(in_s32.val[3], result_fixedpoint_multiplier);
190 
191  // Round to the nearest division by a power-of-two using result_shift_s32
192  in_s32.val[0] = rounding_divide_by_pow2(in_s32.val[0], result_shift);
193  in_s32.val[1] = rounding_divide_by_pow2(in_s32.val[1], result_shift);
194  in_s32.val[2] = rounding_divide_by_pow2(in_s32.val[2], result_shift);
195  in_s32.val[3] = rounding_divide_by_pow2(in_s32.val[3], result_shift);
196  }
197 
198  // Add the offset terms
199  in_s32.val[0] = vaddq_s32(in_s32.val[0], result_offset_after_shift_s32);
200  in_s32.val[1] = vaddq_s32(in_s32.val[1], result_offset_after_shift_s32);
201  in_s32.val[2] = vaddq_s32(in_s32.val[2], result_offset_after_shift_s32);
202  in_s32.val[3] = vaddq_s32(in_s32.val[3], result_offset_after_shift_s32);
203 
204  // Convert S32 to S16
205  const int16x8x2_t in_s16 =
206  {
207  {
208  vcombine_s16(vqmovn_s32(in_s32.val[0]), vqmovn_s32(in_s32.val[1])),
209  vcombine_s16(vqmovn_s32(in_s32.val[2]), vqmovn_s32(in_s32.val[3]))
210  }
211  };
212 
213  // Convert S16 to S8
214  int8x16_t out_s8 = vcombine_s8(vqmovn_s16(in_s16.val[0]), vqmovn_s16(in_s16.val[1]));
215 
216  if(is_bounded_relu)
217  {
218  out_s8 = vmaxq_s8(out_s8, min_s8);
219  out_s8 = vminq_s8(out_s8, max_s8);
220  }
221 
222  return out_s8;
223 }
224 
225 /** Performs final quantization step on 16 elements for symmetric quantization
226  *
227  * @param[in] in_s32 Input to be quantized.
228  * @param[in] result_fixedpoint_multiplier Result multiplier parameter
229  * @param[in] result_shift Result shift parameter
230  * @param[in] result_offset_after_shift_s32 Result offset parameter
231  * @param[in] min_s8 Relu lower bound
232  * @param[in] max_s8 Relu upper bound
233  * @param[in] is_bounded_relu Specified if a fused bounded relu should be applied
234  *
235  * @return Quantized values
236  */
237 inline int8x16_t finalize_quantization_symm(int32x4x4_t &in_s32,
238  const int32x4x4_t &result_fixedpoint_multiplier,
239  const int32x4x4_t &result_shift,
240  const int32x4_t &result_offset_after_shift_s32,
241  const int8x16_t &min_s8,
242  const int8x16_t &max_s8,
243  const bool is_bounded_relu)
244 {
245  const static int32x4_t one_s32 = vdupq_n_s32(1);
246 
247  // Fixed point multiplication with vector saturating rounding doubling multiply high with scalar
248  int32x4x4_t res_shift_gt0 =
249  {
250  vqrdmulhq_s32(in_s32.val[0], result_fixedpoint_multiplier.val[0]),
251  vqrdmulhq_s32(in_s32.val[1], result_fixedpoint_multiplier.val[1]),
252  vqrdmulhq_s32(in_s32.val[2], result_fixedpoint_multiplier.val[2]),
253  vqrdmulhq_s32(in_s32.val[3], result_fixedpoint_multiplier.val[3]),
254  };
255  // Round to the nearest division by a power-of-two using result_shift_s32
256  res_shift_gt0.val[0] = rounding_divide_by_pow2(res_shift_gt0.val[0], result_shift.val[0]);
257  res_shift_gt0.val[1] = rounding_divide_by_pow2(res_shift_gt0.val[1], result_shift.val[1]);
258  res_shift_gt0.val[2] = rounding_divide_by_pow2(res_shift_gt0.val[2], result_shift.val[2]);
259  res_shift_gt0.val[3] = rounding_divide_by_pow2(res_shift_gt0.val[3], result_shift.val[3]);
260 
261  int32x4x4_t res_shift_lt0 =
262  {
263  vmulq_s32(in_s32.val[0], vshlq_s32(one_s32, vnegq_s32(result_shift.val[0]))),
264  vmulq_s32(in_s32.val[1], vshlq_s32(one_s32, vnegq_s32(result_shift.val[1]))),
265  vmulq_s32(in_s32.val[2], vshlq_s32(one_s32, vnegq_s32(result_shift.val[2]))),
266  vmulq_s32(in_s32.val[3], vshlq_s32(one_s32, vnegq_s32(result_shift.val[3]))),
267  };
268  res_shift_lt0.val[0] = vqrdmulhq_s32(res_shift_lt0.val[0], result_fixedpoint_multiplier.val[0]);
269  res_shift_lt0.val[1] = vqrdmulhq_s32(res_shift_lt0.val[1], result_fixedpoint_multiplier.val[1]);
270  res_shift_lt0.val[2] = vqrdmulhq_s32(res_shift_lt0.val[2], result_fixedpoint_multiplier.val[2]);
271  res_shift_lt0.val[3] = vqrdmulhq_s32(res_shift_lt0.val[3], result_fixedpoint_multiplier.val[3]);
272 
273  // Select result depending on shift value
274  const uint32x4x4_t mask_lt0 =
275  {
276 #ifdef __aarch64__
277  vcltzq_s32(result_shift.val[0]),
278  vcltzq_s32(result_shift.val[1]),
279  vcltzq_s32(result_shift.val[2]),
280  vcltzq_s32(result_shift.val[3]),
281 #else //__aarch64__
282  vcltq_s32(result_shift.val[0], vdupq_n_s32(0)),
283  vcltq_s32(result_shift.val[1], vdupq_n_s32(0)),
284  vcltq_s32(result_shift.val[2], vdupq_n_s32(0)),
285  vcltq_s32(result_shift.val[3], vdupq_n_s32(0)),
286 #endif //__aarch64__
287  };
288 
289  in_s32.val[0] = vbslq_s32(mask_lt0.val[0], res_shift_lt0.val[0], res_shift_gt0.val[0]);
290  in_s32.val[1] = vbslq_s32(mask_lt0.val[1], res_shift_lt0.val[1], res_shift_gt0.val[1]);
291  in_s32.val[2] = vbslq_s32(mask_lt0.val[2], res_shift_lt0.val[2], res_shift_gt0.val[2]);
292  in_s32.val[3] = vbslq_s32(mask_lt0.val[3], res_shift_lt0.val[3], res_shift_gt0.val[3]);
293 
294  // Add the offset terms
295  in_s32.val[0] = vaddq_s32(in_s32.val[0], result_offset_after_shift_s32);
296  in_s32.val[1] = vaddq_s32(in_s32.val[1], result_offset_after_shift_s32);
297  in_s32.val[2] = vaddq_s32(in_s32.val[2], result_offset_after_shift_s32);
298  in_s32.val[3] = vaddq_s32(in_s32.val[3], result_offset_after_shift_s32);
299 
300  // Convert S32 to S16
301  const int16x8x2_t in_s16 =
302  {
303  {
304  vcombine_s16(vqmovn_s32(in_s32.val[0]), vqmovn_s32(in_s32.val[1])),
305  vcombine_s16(vqmovn_s32(in_s32.val[2]), vqmovn_s32(in_s32.val[3]))
306  }
307  };
308 
309  // Convert S16 to S8
310  int8x16_t out_s8 = vcombine_s8(vqmovn_s16(in_s16.val[0]), vqmovn_s16(in_s16.val[1]));
311 
312  if(is_bounded_relu)
313  {
314  out_s8 = vmaxq_s8(out_s8, min_s8);
315  out_s8 = vminq_s8(out_s8, max_s8);
316  }
317 
318  return out_s8;
319 }
320 
321 /** Performs final quantization step on single element
322  *
323  * @param[in] in_value Input to be quantized.
324  * @param[in] result_fixedpoint_multiplier Result multiplier parameter
325  * @param[in] result_shift Result shift parameter
326  * @param[in] result_offset_after_shift_s32 Result offset parameter
327  * @param[in] min_u8 Relu lower bound
328  * @param[in] max_u8 Relu upper bound
329  * @param[in] is_bounded_relu Specified if a fused bounded relu should be applied
330  *
331  * @return Quantized value
332  */
333 inline uint8_t finalize_quantization(int32_t in_value, int result_fixedpoint_multiplier,
334  int32_t result_shift, int32_t result_offset_after_shift_s32,
335  uint8_t min_u8, uint8_t max_u8, bool is_bounded_relu)
336 {
337  int32x4_t in_s32 = vdupq_n_s32(in_value);
338 
339  if(result_shift < 0)
340  {
341  in_value = vgetq_lane_s32(vqrdmulhq_n_s32(vmulq_n_s32(in_s32, (1 << (-result_shift))), result_fixedpoint_multiplier), 0);
342  }
343  else
344  {
345  // Fixed point multiplication with vector saturating rounding doubling multiply high with scalar
346  in_value = vgetq_lane_s32(vqrdmulhq_n_s32(in_s32, result_fixedpoint_multiplier), 0);
347  // Shift value by result_shift_s32
348  in_value = rounding_divide_by_pow2(in_value, result_shift);
349  }
350 
351  // Add the offset term
352  in_value += result_offset_after_shift_s32;
353 
354  // Bound the result
355  uint8_t out_u8 = static_cast<uint8_t>(std::max<int32_t>(0, std::min<int32_t>(255, in_value)));
356  if(is_bounded_relu)
357  {
358  out_u8 = static_cast<uint8_t>(std::max(min_u8, std::min(max_u8, out_u8)));
359  }
360 
361  return out_u8;
362 }
363 
364 /** Performs final quantization step on single element
365  *
366  * @param[in] in_value Input to be quantized.
367  * @param[in] result_fixedpoint_multiplier Result multiplier parameter
368  * @param[in] result_shift Result shift parameter
369  * @param[in] result_offset_after_shift_s32 Result offset parameter
370  * @param[in] min_s8 Relu lower bound
371  * @param[in] max_s8 Relu upper bound
372  * @param[in] is_bounded_relu Specified if a fused bounded relu should be applied
373  *
374  * @return Quantized value
375  */
376 inline int8_t finalize_quantization(int32_t in_value, int result_fixedpoint_multiplier,
377  int32_t result_shift, int32_t result_offset_after_shift_s32,
378  int8_t min_s8, int8_t max_s8, bool is_bounded_relu)
379 {
380  int32x4_t in_s32 = vdupq_n_s32(in_value);
381 
382  if(result_shift < 0)
383  {
384  in_value = vgetq_lane_s32(vqrdmulhq_n_s32(vmulq_n_s32(in_s32, (1 << (-result_shift))), result_fixedpoint_multiplier), 0);
385  }
386  else
387  {
388  // Fixed point multiplication with vector saturating rounding doubling multiply high with scalar
389  in_value = vgetq_lane_s32(vqrdmulhq_n_s32(in_s32, result_fixedpoint_multiplier), 0);
390 
391  // Shift value by result_shift_s32
392  in_value = rounding_divide_by_pow2(in_value, result_shift);
393  }
394 
395  // Add the offset term
396  in_value += result_offset_after_shift_s32;
397 
398  // Bound the result
399  int8_t out_s8 = static_cast<int8_t>(std::max<int32_t>(-128, std::min<int32_t>(127, in_value)));
400  if(is_bounded_relu)
401  {
402  out_s8 = static_cast<int8_t>(std::max(min_s8, std::min(max_s8, out_s8)));
403  }
404 
405  return out_s8;
406 }
407 
408 /** Dequantize a neon vector holding 8 quantized values.
409  *
410  * @param[in] qv Input values to be dequantized.
411  * @param[in] qi Quantization information to be used in the computation.
412  *
413  * @return Dequantized values in a neon vector
414  */
415 inline float32x4x2_t vdequantize(const uint8x8_t &qv, const UniformQuantizationInfo &qi)
416 {
417  const float scale = qi.scale;
418  const int offset = qi.offset;
419  const int32x4_t voffset = vdupq_n_s32(offset);
420  const float32x4_t vscale = vdupq_n_f32(scale);
421  const float32x4x2_t vdequantized_input =
422  {
423  {
424  vmulq_f32(vcvtq_f32_s32(vsubq_s32(vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(vmovl_u8(qv)))), voffset)), vscale),
425  vmulq_f32(vcvtq_f32_s32(vsubq_s32(vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(vmovl_u8(qv)))), voffset)), vscale),
426  }
427  };
428  return vdequantized_input;
429 }
430 
431 /** Dequantize a neon vector holding 8 singed quantized values.
432  *
433  * @param[in] qv Input values to be dequantized.
434  * @param[in] qi Quantization information to be used in the computation.
435  *
436  * @return Dequantized values in a neon vector
437  */
438 inline float32x4x2_t vdequantize(const int8x8_t &qv, const UniformQuantizationInfo &qi)
439 {
440  const float scale = qi.scale;
441  const int offset = qi.offset;
442  const int32x4_t voffset = vdupq_n_s32(offset);
443  const float32x4_t vscale = vdupq_n_f32(scale);
444  const float32x4x2_t vdequantized_input =
445  {
446  {
447  vmulq_f32(vcvtq_f32_s32(vsubq_s32(vmovl_s16(vget_low_s16(vmovl_s8(qv))), voffset)), vscale),
448  vmulq_f32(vcvtq_f32_s32(vsubq_s32(vmovl_s16(vget_high_s16(vmovl_s8(qv))), voffset)), vscale),
449  }
450  };
451  return vdequantized_input;
452 }
453 
454 /** Dequantize a neon vector holding 16 quantized values.
455  *
456  * @param[in] qv Input values to be dequantized.
457  * @param[in] qi Quantization information to be used in the computation.
458  *
459  * @return Dequantized values in a neon vector
460  */
461 inline float32x4x4_t vdequantize(const uint8x16_t &qv, const UniformQuantizationInfo &qi)
462 {
463  const float scale = qi.scale;
464  const int offset = qi.offset;
465  const int32x4_t voffset = vdupq_n_s32(offset);
466  const float32x4_t vscale = vdupq_n_f32(scale);
467  const float32x4x4_t vdequantized_input =
468  {
469  {
470  vmulq_f32(vcvtq_f32_s32(vsubq_s32(vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(qv))))), voffset)), vscale),
471  vmulq_f32(vcvtq_f32_s32(vsubq_s32(vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(vmovl_u8(vget_low_u8(qv))))), voffset)), vscale),
472  vmulq_f32(vcvtq_f32_s32(vsubq_s32(vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(vmovl_u8(vget_high_u8(qv))))), voffset)), vscale),
473  vmulq_f32(vcvtq_f32_s32(vsubq_s32(vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(vmovl_u8(vget_high_u8(qv))))), voffset)), vscale),
474  }
475  };
476  return vdequantized_input;
477 }
478 
479 /** Dequantize a neon vector holding 16 signed quantized values.
480  *
481  * @param[in] qv Input values to be dequantized.
482  * @param[in] qi Quantization information to be used in the computation.
483  *
484  * @return Dequantized values in a neon vector
485  */
486 inline float32x4x4_t vdequantize(const int8x16_t &qv, const UniformQuantizationInfo &qi)
487 {
488  const float scale = qi.scale;
489  const int offset = qi.offset;
490  const int32x4_t voffset = vdupq_n_s32(offset);
491  const float32x4_t vscale = vdupq_n_f32(scale);
492  const float32x4x4_t vdequantized_input =
493  {
494  {
495  vmulq_f32(vcvtq_f32_s32(vsubq_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_low_s8(qv)))), voffset)), vscale),
496  vmulq_f32(vcvtq_f32_s32(vsubq_s32(vmovl_s16(vget_high_s16(vmovl_s8(vget_low_s8(qv)))), voffset)), vscale),
497  vmulq_f32(vcvtq_f32_s32(vsubq_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_high_s8(qv)))), voffset)), vscale),
498  vmulq_f32(vcvtq_f32_s32(vsubq_s32(vmovl_s16(vget_high_s16(vmovl_s8(vget_high_s8(qv)))), voffset)), vscale),
499  }
500  };
501  return vdequantized_input;
502 }
503 
504 /** Dequantize following an asymmetric quantization scheme a neon vector holding 16 quantized values.
505  *
506  * @param[in] qv Input values to be dequantized.
507  * @param[in] scale Quantization scaling factor.
508  * @param[in] offset Zero quantization offset.
509  *
510  * @return Dequantized values in a neon vector
511  */
512 inline float32x4x4_t vdequantize(const uint8x16_t &qv, float scale, int32_t offset)
513 {
514  const int32x4_t voffset = vdupq_n_s32(offset);
515  const float32x4_t vscale = vdupq_n_f32(scale);
516  const float32x4x4_t vdequantized_input =
517  {
518  {
519  vmulq_f32(vcvtq_f32_s32(vsubq_s32(vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(qv))))), voffset)), vscale),
520  vmulq_f32(vcvtq_f32_s32(vsubq_s32(vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(vmovl_u8(vget_low_u8(qv))))), voffset)), vscale),
521  vmulq_f32(vcvtq_f32_s32(vsubq_s32(vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(vmovl_u8(vget_high_u8(qv))))), voffset)), vscale),
522  vmulq_f32(vcvtq_f32_s32(vsubq_s32(vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(vmovl_u8(vget_high_u8(qv))))), voffset)), vscale),
523  }
524  };
525  return vdequantized_input;
526 }
527 
528 /** Dequantize a vector of 16 values stored as signed asymmetric.
529  *
530  * @param[in] qv Input values to be dequantized.
531  * @param[in] scale Quantization scaling factor.
532  * @param[in] offset Zero quantization offset.
533  *
534  * @return Dequantized values in a neon vector
535  */
536 inline float32x4x4_t vdequantize(const int8x16_t &qv, float scale, int32_t offset)
537 {
538  const int32x4_t voffset = vdupq_n_s32(offset);
539  const float32x4_t vscale = vdupq_n_f32(scale);
540  const float32x4x4_t vdequantized_input =
541  {
542  {
543  vmulq_f32(vcvtq_f32_s32(vsubq_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_low_s8(qv)))), voffset)), vscale),
544  vmulq_f32(vcvtq_f32_s32(vsubq_s32(vmovl_s16(vget_high_s16(vmovl_s8(vget_low_s8(qv)))), voffset)), vscale),
545  vmulq_f32(vcvtq_f32_s32(vsubq_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_high_s8(qv)))), voffset)), vscale),
546  vmulq_f32(vcvtq_f32_s32(vsubq_s32(vmovl_s16(vget_high_s16(vmovl_s8(vget_high_s8(qv)))), voffset)), vscale),
547  }
548  };
549  return vdequantized_input;
550 }
551 
552 /** Dequantize following symmetric quantization scheme a neon vector holding 16 quantized values.
553  *
554  * @param[in] qv Input values to be dequantized.
555  * @param[in] vscale Vector containing quantization scaling factors.
556  *
557  * @return Dequantized values in a neon vector
558  */
559 inline float32x4x4_t vdequantize(const int8x16_t &qv, const float32x4x4_t vscale)
560 {
561  const float32x4x4_t vdequantized_input =
562  {
563  {
564  vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_low_s8(qv))))), vscale.val[0]),
565  vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_high_s16(vmovl_s8(vget_low_s8(qv))))), vscale.val[1]),
566  vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_high_s8(qv))))), vscale.val[2]),
567  vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_high_s16(vmovl_s8(vget_high_s8(qv))))), vscale.val[3]),
568  }
569  };
570  return vdequantized_input;
571 }
572 
573 /** Dequantize following a symmetric quantization scheme a neon vector holding 16 quantized values.
574  *
575  * @param[in] qv Input values to be dequantized.
576  * @param[in] scale Quantization scaling factor.
577  *
578  * @return Dequantized values in a neon vector
579  */
580 inline float32x4x4_t vdequantize(const int8x16_t &qv, float scale)
581 {
582  const float32x4_t vscale = vdupq_n_f32(scale);
583  const float32x4x4_t vdequantized_input =
584  {
585  {
586  vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_low_s8(qv))))), vscale),
587  vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_high_s16(vmovl_s8(vget_low_s8(qv))))), vscale),
588  vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_high_s8(qv))))), vscale),
589  vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_high_s16(vmovl_s8(vget_high_s8(qv))))), vscale),
590  }
591  };
592  return vdequantized_input;
593 }
594 
595 /** Quantize a neon vector holding 8 floating point values.
596  *
597  * @param[in] qv Input values to be quantized.
598  * @param[in] qi Quantization information to be used in the computation.
599  *
600  * @return A neon vector holding the quantized values
601  */
602 inline uint8x8_t vquantize(const float32x4x2_t &qv, const UniformQuantizationInfo &qi)
603 {
604  const float scale = qi.scale;
605  const int offset = qi.offset;
606  const float32x4_t voffset = vdupq_n_f32(offset);
607  const float32x4_t vinvscale = vdupq_n_f32(1.f / scale);
608  const int32x4x4_t rf =
609  {
610  {
611 #ifdef __aarch64__
612  vcvtnq_s32_f32(vmlaq_f32(voffset, qv.val[0], vinvscale)),
613  vcvtnq_s32_f32(vmlaq_f32(voffset, qv.val[1], vinvscale)),
614 #else //__aarch64__
615  vcvtq_s32_f32(vmlaq_f32(voffset, qv.val[0], vinvscale)),
616  vcvtq_s32_f32(vmlaq_f32(voffset, qv.val[1], vinvscale)),
617 #endif //__aarch64__
618  }
619  };
620  return vqmovun_s16(vcombine_s16(vqmovn_s32(rf.val[0]), vqmovn_s32(rf.val[1])));
621 }
622 
623 /** Quantize a neon vector holding 8 floating point values.
624  *
625  * @param[in] qv Input values to be quantized.
626  * @param[in] qi Quantization information to be used in the computation.
627  *
628  * @return A neon vector holding the singed quantized values
629  */
630 inline int8x8_t vquantize_signed(const float32x4x2_t &qv, const UniformQuantizationInfo &qi)
631 {
632  const float scale = qi.scale;
633  const int offset = qi.offset;
634  const float32x4_t voffset = vdupq_n_f32(offset);
635  const float32x4_t vinvscale = vdupq_n_f32(1.f / scale);
636  const int32x4x4_t rf =
637  {
638  {
639 #ifdef __aarch64__
640  vcvtnq_s32_f32(vmlaq_f32(voffset, qv.val[0], vinvscale)),
641  vcvtnq_s32_f32(vmlaq_f32(voffset, qv.val[1], vinvscale)),
642 #else //__aarch64__
643  vcvtq_s32_f32(vmlaq_f32(voffset, qv.val[0], vinvscale)),
644  vcvtq_s32_f32(vmlaq_f32(voffset, qv.val[1], vinvscale)),
645 #endif //__aarch64__
646  }
647  };
648  return vqmovn_s16(vcombine_s16(vqmovn_s32(rf.val[0]), vqmovn_s32(rf.val[1])));
649 }
650 
651 inline int32x4x4_t vquantize_internal(const float32x4x4_t &qv, float scale, int32_t offset)
652 {
653  const int32x4_t voffset = vdupq_n_s32(offset);
654  const float32x4_t vinvscale = vdupq_n_f32(1.f / scale);
655  const int32x4x4_t rf =
656  {
657  {
658 #ifdef __aarch64__
659  vaddq_s32(vcvtaq_s32_f32(vmulq_f32(qv.val[0], vinvscale)), voffset),
660  vaddq_s32(vcvtaq_s32_f32(vmulq_f32(qv.val[1], vinvscale)), voffset),
661  vaddq_s32(vcvtaq_s32_f32(vmulq_f32(qv.val[2], vinvscale)), voffset),
662  vaddq_s32(vcvtaq_s32_f32(vmulq_f32(qv.val[3], vinvscale)), voffset),
663 #else //__aarch64__
664  vaddq_s32(vcvtq_s32_f32(vmulq_f32(qv.val[0], vinvscale)), voffset),
665  vaddq_s32(vcvtq_s32_f32(vmulq_f32(qv.val[1], vinvscale)), voffset),
666  vaddq_s32(vcvtq_s32_f32(vmulq_f32(qv.val[2], vinvscale)), voffset),
667  vaddq_s32(vcvtq_s32_f32(vmulq_f32(qv.val[3], vinvscale)), voffset),
668 #endif //__aarch64__
669  }
670  };
671  return rf;
672 }
673 
674 /** Quantize a neon vector holding 16 floating point values.
675  *
676  * @param[in] qv Input values to be quantized.
677  * @param[in] qi Quantization information to be used in the computation.
678  *
679  * @return A neon vector holding the quantized values
680  */
681 inline uint8x16_t vquantize(const float32x4x4_t &qv, const UniformQuantizationInfo &qi)
682 {
683  auto rf = vquantize_internal(qv, qi.scale, qi.offset);
684  const uint8x8_t pa = vqmovun_s16(vcombine_s16(vqmovn_s32(rf.val[0]), vqmovn_s32(rf.val[1])));
685  const uint8x8_t pb = vqmovun_s16(vcombine_s16(vqmovn_s32(rf.val[2]), vqmovn_s32(rf.val[3])));
686  return vcombine_u8(pa, pb);
687 }
688 
689 /** Signed quantize a neon vector holding 16 floating point values.
690  *
691  * @param[in] qv Input values to be quantized.
692  * @param[in] qi Quantization information to be used in the computation.
693  *
694  * @return A neon vector holding the quantized values
695  */
696 inline int8x16_t vquantize_signed(const float32x4x4_t &qv, const UniformQuantizationInfo &qi)
697 {
698  auto rf = vquantize_internal(qv, qi.scale, qi.offset);
699  const int8x8_t pa = vqmovn_s16(vcombine_s16(vqmovn_s32(rf.val[0]), vqmovn_s32(rf.val[1])));
700  const int8x8_t pb = vqmovn_s16(vcombine_s16(vqmovn_s32(rf.val[2]), vqmovn_s32(rf.val[3])));
701  return vcombine_s8(pa, pb);
702 }
703 
704 /** Quantize to QASYMM16 a neon vector holding 16 floating point values.
705  *
706  * @param[in] qv Input values to be quantized.
707  * @param[in] qi Quantization information to be used in the computation.
708  *
709  * @return A neon vector holding the quantized values
710  */
711 inline uint16x8x2_t vquantize_qasymm16(const float32x4x4_t &qv, const UniformQuantizationInfo &qi)
712 {
713  auto rf = vquantize_internal(qv, qi.scale, qi.offset);
714  const uint16x8_t pa = vcombine_u16(vqmovun_s32(rf.val[0]), vqmovun_s32(rf.val[1]));
715  const uint16x8_t pb = vcombine_u16(vqmovun_s32(rf.val[2]), vqmovun_s32(rf.val[3]));
716  return { pa, pb };
717 }
718 } // namespace arm_compute
719 #include "src/core/NEON/NEAsymm.inl"
720 #endif // ARM_COMPUTE_NEASYMM_H
__global uchar * offset(const Image *img, int x, int y)
Get the pointer position of a Image.
Definition: helpers.h:846
uint8x8x2_t qasymm8x8x2_t
8 bit quantized asymmetric vector with 16 elements
Definition: NEAsymm.h:34
uint16x8x2_t vquantize_qasymm16(const float32x4x4_t &qv, const UniformQuantizationInfo &qi)
Quantize to QASYMM16 a neon vector holding 16 floating point values.
Definition: NEAsymm.h:711
float32x4x2_t vdequantize(const uint8x8_t &qv, const UniformQuantizationInfo &qi)
Dequantize a neon vector holding 8 quantized values.
Definition: NEAsymm.h:415
int8x8x4_t qasymm8x8x4_signed_t
8 bit quantized signed asymmetric vector with 32 elements
Definition: NEAsymm.h:42
Quantization info when assuming per layer quantization.
Copyright (c) 2017-2021 Arm Limited.
uint8x16_t vmlaq_qasymm8(qasymm8x16_t vd, float32x4_t vs, float32x4_t vo)
Perform a multiply-accumulate on all 16 components of a QASYMM8 vector.
Definition: NEAsymm.inl:26
int8x8_t qasymm8x8_signed_t
8 bit quantized signed asymmetric vector with 8 elements
Definition: NEAsymm.h:39
int8x8x2_t qasymm8x8x2_signed_t
8 bit quantized signed asymmetric vector with 16 elements
Definition: NEAsymm.h:40
uint8x8x3_t qasymm8x8x3_t
8 bit quantized asymmetric vector with 24 elements
Definition: NEAsymm.h:35
int8x8x3_t qasymm8x8x3_signed_t
8 bit quantized signed asymmetric vector with 24 elements
Definition: NEAsymm.h:41
uint8x8x4_t qasymm8x8x4_t
8 bit quantized asymmetric vector with 32 elements
Definition: NEAsymm.h:36
uint8x8_t qasymm8x8_t
8 bit quantized asymmetric vector with 8 elements
Definition: NEAsymm.h:33
int8x16_t vmlaq_qasymm8_signed(qasymm8x16_signed_t vd, float32x4_t vs, float32x4_t vo)
Perform a multiply-accumulate on all 16 components of a QASYMM8_SIGNED vector.
Definition: NEAsymm.inl:59
int32x4_t rounding_divide_by_pow2(int32x4_t x, int32x4_t exponent)
Round to the nearest division by a power-of-two using exponent.
Definition: NEMath.inl:299
uint8x8_t vquantize(const float32x4x2_t &qv, const UniformQuantizationInfo &qi)
Quantize a neon vector holding 8 floating point values.
Definition: NEAsymm.h:602
int8x16_t finalize_quantization_symm(int32x4x4_t &in_s32, const int32x4x4_t &result_fixedpoint_multiplier, const int32x4x4_t &result_shift, const int32x4_t &result_offset_after_shift_s32, const int8x16_t &min_s8, const int8x16_t &max_s8, const bool is_bounded_relu)
Performs final quantization step on 16 elements for symmetric quantization.
Definition: NEAsymm.h:237
int8x8_t vquantize_signed(const float32x4x2_t &qv, const UniformQuantizationInfo &qi)
Quantize a neon vector holding 8 floating point values.
Definition: NEAsymm.h:630
int8x16_t qasymm8x16_signed_t
8 bit quantized signed asymmetric vector with 16 elements
Definition: NEAsymm.h:43
wrapper::traits::neon_vector< T, 16 >::type finalize_quantization(int32x4x4_t &in_s32, int32x4_t result_shift_s32, typename wrapper::traits::neon_vector< T, 16 >::type min, typename wrapper::traits::neon_vector< T, 16 >::type max)
int32x4x4_t vquantize_internal(const float32x4x4_t &qv, float scale, int32_t offset)
Definition: NEAsymm.h:651
uint8x16_t qasymm8x16_t
8 bit quantized asymmetric vector with 16 elements
Definition: NEAsymm.h:37