arm_compute::cpu Namespace Reference

Namespaces
	kernel
	kernels

Data Structures
struct	ActFpImplParams
	Constant parameters needed by the activation implementation. More...
struct	AsmGemmInfo
class	CpuActivation
	Basic function to run kernels::CpuActivationKernel. More...
class	CpuAdd
	Basic function to run kernels::CpuAddKernel. More...
class	CpuAuxTensorHandler
struct	CpuCapabilities
	Structure that encodes the CPU capabilities to be used. More...
class	CpuCast
	Basic function to run kernels::CpuCastKernel. More...
class	CpuComplexMul
	Basic function to run kernels::CpuComplexMulKernel. More...
class	CpuConcatenate
	Basic function to execute concatenate tensors along a given axis. More...
class	CpuContext
	CPU context implementation class. More...
class	CpuConv2d
	Basic function to simulate a convolution layer. More...
class	CpuConvertFullyConnectedWeights
	Basic function to run kernels::CpuConvertFullyConnectedWeightsKernel. More...
class	CpuCopy
	Basic function to run kernels::CpuCopyKernel. More...
class	CpuDepthwiseConv2d
	Function to execute a depthwise convolution. More...
class	CpuDepthwiseConv2dAssemblyDispatch
	Depthwise convolution assembly kernel glue. More...
class	CpuDequantize
	Basic function to run kernels::CpuDequantizeKernel that dequantizes an input tensor. More...
class	CpuDirectConv2d
	Function to run the direct convolution. More...
class	CpuDirectConv3d
	Function to run the direct convolution. More...
class	CpuElementwiseArithmetic
	Class to run cpu::kernels::CpuArithmeticKernel except for division and power. More...
class	CpuElementwiseBase
class	CpuElementwiseComparison
	Basic function to run cpu::kernels::CpuComparisonKernel. More...
class	CpuElementwiseComparisonStatic
	Basic function to run cpu::kernels::CpuComparisonKernel. More...
class	CpuElementwiseDivision
	Basic function to run cpu::kernels::CpuArithmeticKernel for division. More...
class	CpuElementwisePower
	Basic function to run cpu::kernels::CpuArithmeticKernel for power. More...
class	CpuElementwiseUnary
class	CpuFill
	Basic function to run kernels::CpuFillKernel. More...
class	CpuFlatten
	Basic function to flatten a given input. More...
class	CpuFloor
	Basic function to run kernels::CpuFloorKernel. More...
class	CpuFullyConnected
	Basic function to compute a Fully Connected layer. More...
class	CpuGemm
	Basic function to execute GEMM. More...
class	CpuGemmAssemblyDispatch
	Assembly kernel glue. More...
class	CpuGemmConv2d
	Basic function to compute the convolution layer. More...
class	CpuGemmDirectConv2d
class	CpuGemmLowpMatrixMultiplyCore
	Basic function to execute GEMMLowpMatrixMultiplyCore. More...
class	CpuGemmLowpOutputStage
	Basic function to execute GEMMLowpQuantizeDown kernels. More...
class	CpuLogits1DSoftmaxKernel
class	CpuMaxUnpooling
	Basic function to run kernels::CpuMaxUnpoolingLayerKernel. More...
class	CpuMul
	Basic function to run kernels::CpuMulKernel. More...
class	CpuPermute
	Basic function to run kernels::CpuPermuteKernel. More...
class	CpuPool2d
	Basic function to simulate a pooling layer with the specified pooling operation. More...
class	CpuPool3d
	Basic function to simulate a pooling layer with the specified pooling operation. More...
class	CpuQuantize
	Basic function to run kernels::CpuQuantizeKernel that dequantizes an input tensor. More...
class	CpuQueue
	CPU queue implementation class. More...
class	CpuReshape
	Basic function to run kernels::CpuReshapeKernel. More...
class	CpuScale
	Basic function to compute Scale. More...
class	CpuSoftmaxGeneric
	Basic function to compute a SoftmaxLayer and a Log SoftmaxLayer. More...
class	CpuSub
	Basic function to run kernels::CpuSubKernel. More...
class	CpuTensor
	CPU tensor implementation class. More...
class	CpuTranspose
	Basic function to run kernels::CpuTransposeKernel. More...
class	CpuWinogradConv2d
class	CpuWinogradConv2dTransformInputKernel
class	CpuWinogradConv2dTransformOutputKernel
class	ICpuKernel

Typedefs
using	ICpuOperator = experimental::INEOperator
using	CpuElementwiseMax = CpuElementwiseArithmetic< ArithmeticOperation::MAX >
	Class to run cpu::kernels::CpuArithmeticKernel except for maximum operation. More...
using	CpuElementwiseMin = CpuElementwiseArithmetic< ArithmeticOperation::MIN >
	Class to run cpu::kernels::CpuArithmeticKernel except for minimum operation. More...
using	CpuElementwiseSquaredDiff = CpuElementwiseArithmetic< ArithmeticOperation::SQUARED_DIFF >
	Class to run cpu::kernels::CpuArithmeticKernel except for squared difference operation. More...
using	NEEqual = CpuElementwiseComparisonStatic< ComparisonOperation::Equal >
	Basic function to run equal comparison. More...
using	NENotEqual = CpuElementwiseComparisonStatic< ComparisonOperation::NotEqual >
	Basic function to run not equal comparison. More...
using	NEGreater = CpuElementwiseComparisonStatic< ComparisonOperation::Greater >
	Basic function to run greater comparison. More...
using	NEGreaterEqual = CpuElementwiseComparisonStatic< ComparisonOperation::GreaterEqual >
	Basic function to run greater-equal comparison. More...
using	NELess = CpuElementwiseComparisonStatic< ComparisonOperation::Less >
	Basic function to run less comparison. More...
using	NELessEqual = CpuElementwiseComparisonStatic< ComparisonOperation::LessEqual >
	Basic function to run less-equal comparison. More...
using	KernelType = kernels::CpuElementwiseUnaryKernel
using	CpuPRelu = CpuElementwiseArithmetic< ArithmeticOperation::PRELU >
	Class to run cpu::kernels::CpuArithmeticKernel except for PRelu operation. More...
using	CpuSoftmax = CpuSoftmaxGeneric< false >
using	CpuLogSoftmax = CpuSoftmaxGeneric< true >

Enumerations
enum	KernelSelectionType { Preferred, Supported }
enum	AsmConvMethod { Im2Col, Indirect, Conv }

Functions
void	fp16_neon_batch_normalization (ITensor *src, ITensor *dst, const ITensor *mean, const ITensor *var, const ITensor *beta, const ITensor *gamma, float epsilon, ActivationLayerInfo &act_info, const Window &window)
void	fp16_sve_batch_normalization (ITensor *src, ITensor *dst, const ITensor *mean, const ITensor *var, const ITensor *beta, const ITensor *gamma, float epsilon, ActivationLayerInfo &act_info, const Window &window)
void	fp32_neon_batch_normalization (ITensor *src, ITensor *dst, const ITensor *mean, const ITensor *var, const ITensor *beta, const ITensor *gamma, float epsilon, ActivationLayerInfo &act_info, const Window &window)
void	fp32_sve_batch_normalization (ITensor *src, ITensor *dst, const ITensor *mean, const ITensor *var, const ITensor *beta, const ITensor *gamma, float epsilon, ActivationLayerInfo &act_info, const Window &window)
void	neon_fp32_activation (const ITensor *src, ITensor *dst, const ActivationLayerInfo &act_info, const Window &window)
float32x4_t	mask_float_vector (const float32x4_t &in, const uint32x4_t &mask)
template<typename T , const ActFpImplParams & P>
void	fp_neon_activation_impl (const ITensor *src, ITensor *dst, const ActivationLayerInfo &act_info, const Window &window)
void	neon_qasymm8_activation_lut (const ITensor *src, ITensor *dst, const ActivationLayerInfo &act_info, const Window &window)
void	neon_qasymm8_activation (const ITensor *src, ITensor *dst, const ActivationLayerInfo &act_info, const Window &window)
void	neon_qasymm8_signed_activation (const ITensor *src, ITensor *dst, const ActivationLayerInfo &act_info, const Window &window)
void	neon_qsymm16_activation (const ITensor *src, ITensor *dst, const ActivationLayerInfo &act_info, const Window &window)
void	sve_fp32_activation (const ITensor *src, ITensor *dst, const ActivationLayerInfo &act_info, const Window &window)
void	sve2_qasymm8_activation (const ITensor *src, ITensor *dst, const ActivationLayerInfo &act_info, const Window &window)
void	sve2_qasymm8_signed_activation (const ITensor *src, ITensor *dst, const ActivationLayerInfo &act_info, const Window &window)
void	sve2_qsymm16_activation (const ITensor *src, ITensor *dst, const ActivationLayerInfo &act_info, const Window &window)
void	sve_fp16_activation (const ITensor *src, ITensor *dst, const ActivationLayerInfo &act_info, const Window &window)
void	neon_fp16_activation (const ITensor *src, ITensor *dst, const ActivationLayerInfo &act_info, const Window &window)
void	add_fp32_neon (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
void	add_fp32_neon_as_1d_array (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
template<typename ScalarType >
void	add_same_neon (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
template<typename ScalarType >
void	add_same_neon_as_1d_array (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
template void	add_same_neon< float > (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
template void	add_same_neon< uint8_t > (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
template void	add_same_neon< int32_t > (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
template void	add_same_neon< int16_t > (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
template void	add_same_neon_as_1d_array< float > (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
template void	add_same_neon_as_1d_array< uint8_t > (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
template void	add_same_neon_as_1d_array< int32_t > (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
template void	add_same_neon_as_1d_array< int16_t > (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
void	add_u8_neon (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
void	add_s16_neon (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
void	add_s32_neon (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
void	add_u8_neon_as_1d_array (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
void	add_s16_neon_as_1d_array (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
void	add_s32_neon_as_1d_array (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
void	add_qasymm8_neon (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
void	add_qasymm8_signed_neon (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
void	add_qsymm16_neon (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
void	add_fp32_sve (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
template<typename ScalarType >
void	add_same_sve (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
template void	add_same_sve< float > (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
template void	add_same_sve< uint8_t > (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
template void	add_same_sve< int16_t > (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
template void	add_same_sve< int32_t > (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
void	add_u8_sve (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
void	add_s16_sve (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
void	add_s32_sve (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
void	add_qasymm8_sve2 (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
void	add_qasymm8_signed_sve2 (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
void	add_qsymm16_sve2 (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
void	add_fp16_neon (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
void	add_fp16_neon_as_1d_array (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
void	add_fp16_sve (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
void	neon_fp32_boundingboxtransform (const ITensor *boxes, ITensor *pred_boxes, const ITensor *deltas, BoundingBoxTransformInfo bbinfo, const Window &window)
void	bounding_box_transform_qsymm16 (const ITensor *boxes, ITensor *pred_boxes, const ITensor *deltas, BoundingBoxTransformInfo bbinfo, const Window &window)
template<typename T >
void	bounding_box_transform (const ITensor *boxes, ITensor *pred_boxes, const ITensor *deltas, BoundingBoxTransformInfo bbinfo, const Window &window)
template void	bounding_box_transform< float > (const ITensor *boxes, ITensor *pred_boxes, const ITensor *deltas, BoundingBoxTransformInfo bbinfo, const Window &window)
void	neon_qu16_boundingboxtransform (const ITensor *boxes, ITensor *pred_boxes, const ITensor *deltas, BoundingBoxTransformInfo bbinfo, const Window &window)
void	neon_fp16_boundingboxtransform (const ITensor *boxes, ITensor *pred_boxes, const ITensor *deltas, BoundingBoxTransformInfo bbinfo, const Window &window)
void	neon_fp32_to_fp16_cast (const ITensor *_src, ITensor *_dst, const ThreadInfo &tensor, ConvertPolicy _policy, const Window &window)
void	neon_u8_to_fp16_cast (const ITensor *_src, ITensor *_dst, const ThreadInfo &tensor, ConvertPolicy _policy, const Window &window)
void	neon_fp16_to_other_dt_cast (const ITensor *_src, ITensor *_dst, const ThreadInfo &tensor, ConvertPolicy _policy, const Window &window)
void	neon_s32_to_fp16_cast (const ITensor *_src, ITensor *_dst, const ThreadInfo &tensor, ConvertPolicy _policy, const Window &window)
void	neon_qasymm8_signed_to_fp16_cast (const ITensor *_src, ITensor *_dst, const ThreadInfo &tensor, ConvertPolicy _policy, const Window &window)
void	neon_fp32_to_bfloat16_cast (const ITensor *_src, ITensor *_dst, const ThreadInfo &tensor, ConvertPolicy _policy, const Window &window)
void	neon_bfloat16_to_fp32_cast (const ITensor *_src, ITensor *_dst, const ThreadInfo &tensor, ConvertPolicy _policy, const Window &window)
template<typename T >
void	directconv3d_float_neon_ndhwc (const ITensor *src0, const ITensor *src1, const ITensor *src2, ITensor *dst, const Conv3dInfo &conv_info, const Window &window)
template<typename T >
void	directconv3d_quantized_neon_ndhwc (const ITensor *src0, const ITensor *src1, const ITensor *src2, ITensor *dst, const Conv3dInfo &conv_info, const Window &window)
template<typename T >
float32x4_t	load_as_f32 (T *ptr)
template<>
float32x4_t	load_as_f32 (float *ptr)
template<>
float32x4_t	load_as_f32 (int32_t *ptr)
template<>
float32x4_t	load_as_f32 (uint32_t *ptr)
template<>
float32x4_t	load_as_f32 (int16_t *ptr)
template<>
float32x4_t	load_as_f32 (uint16_t *ptr)
template<>
float32x4_t	load_as_f32 (uint8_t *ptr)
void	fp32_in_bounds_crop_window (const ITensor *input, const ITensor *output, float *output_ptr, Coordinates input_offset, int32_t window_step_x, int32_t output_width_start, int32_t output_width_limit, bool input_has_single_channel, bool is_width_flipped)
template<typename T >
void	in_bounds_crop_window (const ITensor *input, const ITensor *output, float *output_ptr, Coordinates input_offset, int32_t window_step_x, int32_t output_width_start, int32_t output_width_limit, bool input_has_single_channel, bool is_width_flipped)
template void	in_bounds_crop_window< float32_t > (const ITensor *input, const ITensor *output, float *output_ptr, Coordinates input_offset, int32_t window_step_x, int32_t output_width_start, int32_t output_width_limit, bool input_has_single_channel, bool is_width_flipped)
template void	in_bounds_crop_window< uint8_t > (const ITensor *input, const ITensor *output, float *output_ptr, Coordinates input_offset, int32_t window_step_x, int32_t output_width_start, int32_t output_width_limit, bool input_has_single_channel, bool is_width_flipped)
template void	in_bounds_crop_window< uint16_t > (const ITensor *input, const ITensor *output, float *output_ptr, Coordinates input_offset, int32_t window_step_x, int32_t output_width_start, int32_t output_width_limit, bool input_has_single_channel, bool is_width_flipped)
template void	in_bounds_crop_window< uint32_t > (const ITensor *input, const ITensor *output, float *output_ptr, Coordinates input_offset, int32_t window_step_x, int32_t output_width_start, int32_t output_width_limit, bool input_has_single_channel, bool is_width_flipped)
template void	in_bounds_crop_window< int8_t > (const ITensor *input, const ITensor *output, float *output_ptr, Coordinates input_offset, int32_t window_step_x, int32_t output_width_start, int32_t output_width_limit, bool input_has_single_channel, bool is_width_flipped)
template void	in_bounds_crop_window< int16_t > (const ITensor *input, const ITensor *output, float *output_ptr, Coordinates input_offset, int32_t window_step_x, int32_t output_width_start, int32_t output_width_limit, bool input_has_single_channel, bool is_width_flipped)
template void	in_bounds_crop_window< int32_t > (const ITensor *input, const ITensor *output, float *output_ptr, Coordinates input_offset, int32_t window_step_x, int32_t output_width_start, int32_t output_width_limit, bool input_has_single_channel, bool is_width_flipped)
void	u8_in_bounds_crop_window (const ITensor *input, const ITensor *output, float *output_ptr, Coordinates input_offset, int32_t window_step_x, int32_t output_width_start, int32_t output_width_limit, bool input_has_single_channel, bool is_width_flipped)
void	u16_in_bounds_crop_window (const ITensor *input, const ITensor *output, float *output_ptr, Coordinates input_offset, int32_t window_step_x, int32_t output_width_start, int32_t output_width_limit, bool input_has_single_channel, bool is_width_flipped)
void	u32_in_bounds_crop_window (const ITensor *input, const ITensor *output, float *output_ptr, Coordinates input_offset, int32_t window_step_x, int32_t output_width_start, int32_t output_width_limit, bool input_has_single_channel, bool is_width_flipped)
void	s8_in_bounds_crop_window (const ITensor *input, const ITensor *output, float *output_ptr, Coordinates input_offset, int32_t window_step_x, int32_t output_width_start, int32_t output_width_limit, bool input_has_single_channel, bool is_width_flipped)
void	s16_in_bounds_crop_window (const ITensor *input, const ITensor *output, float *output_ptr, Coordinates input_offset, int32_t window_step_x, int32_t output_width_start, int32_t output_width_limit, bool input_has_single_channel, bool is_width_flipped)
void	s32_in_bounds_crop_window (const ITensor *input, const ITensor *output, float *output_ptr, Coordinates input_offset, int32_t window_step_x, int32_t output_width_start, int32_t output_width_limit, bool input_has_single_channel, bool is_width_flipped)
void	fp16_in_bounds_crop_window (const ITensor *input, const ITensor *output, float *output_ptr, Coordinates input_offset, int32_t window_step_x, int32_t output_width_start, int32_t output_width_limit, bool input_has_single_channel, bool is_width_flipped)
void	neon_fp32_deptwiseconv2dnative (const ITensor *src, const ITensor *weights, const ITensor *bias, ITensor *dst, const Window &window, bool has_biases, const ConvolutionInfo &info)
template<typename T , typename TW >
void	run_depthwise_float (const ITensor *src, const ITensor *weights, const ITensor *biases, ITensor *dst, const Window &window, bool has_biases, const ConvolutionInfo &info)
template void	run_depthwise_float< float, float > (const ITensor *src, const ITensor *weights, const ITensor *biases, ITensor *dst, const Window &window, bool has_biases, const ConvolutionInfo &info)
template<typename T , typename TW >
void	run_depthwise_quanitized8bit (const ITensor *src, const ITensor *weights, const ITensor *biases, ITensor *dst, const Window &window, bool has_biases, const ConvolutionInfo &info)
template void	run_depthwise_quanitized8bit< uint8_t, uint8_t > (const ITensor *src, const ITensor *weights, const ITensor *biases, ITensor *dst, const Window &window, bool has_biases, const ConvolutionInfo &info)
template void	run_depthwise_quanitized8bit< int8_t, int8_t > (const ITensor *src, const ITensor *weights, const ITensor *biases, ITensor *dst, const Window &window, bool has_biases, const ConvolutionInfo &info)
template void	run_depthwise_quanitized8bit< uint8_t, int8_t > (const ITensor *src, const ITensor *weights, const ITensor *biases, ITensor *dst, const Window &window, bool has_biases, const ConvolutionInfo &info)
void	neon_qu8_deptwiseconv2dnative (const ITensor *src, const ITensor *weights, const ITensor *bias, ITensor *dst, const Window &window, bool has_biases, const ConvolutionInfo &info)
void	neon_qp8_qu8_deptwiseconv2dnative (const ITensor *src, const ITensor *weights, const ITensor *bias, ITensor *dst, const Window &window, bool has_biases, const ConvolutionInfo &info)
void	neon_qs8_deptwiseconv2dnative (const ITensor *src, const ITensor *weights, const ITensor *bias, ITensor *dst, const Window &window, bool has_biases, const ConvolutionInfo &info)
void	neon_qp8_qs8_deptwiseconv2dnative (const ITensor *src, const ITensor *weights, const ITensor *bias, ITensor *dst, const Window &window, bool has_biases, const ConvolutionInfo &info)
void	neon_fp16_deptwiseconv2dnative (const ITensor *src, const ITensor *weights, const ITensor *bias, ITensor *dst, const Window &window, bool has_biases, const ConvolutionInfo &info)
template<ArithmeticOperation op>
void	neon_fp32_elementwise_binary (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_fp32_elementwise_binary< ArithmeticOperation::ADD > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_fp32_elementwise_binary< ArithmeticOperation::SUB > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_fp32_elementwise_binary< ArithmeticOperation::DIV > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_fp32_elementwise_binary< ArithmeticOperation::MIN > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_fp32_elementwise_binary< ArithmeticOperation::MAX > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_fp32_elementwise_binary< ArithmeticOperation::SQUARED_DIFF > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_fp32_elementwise_binary< ArithmeticOperation::POWER > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_fp32_elementwise_binary< ArithmeticOperation::PRELU > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template<ComparisonOperation op>
void	neon_fp32_comparison_elementwise_binary (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_fp32_comparison_elementwise_binary< ComparisonOperation::Equal > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_fp32_comparison_elementwise_binary< ComparisonOperation::NotEqual > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_fp32_comparison_elementwise_binary< ComparisonOperation::Greater > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_fp32_comparison_elementwise_binary< ComparisonOperation::GreaterEqual > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_fp32_comparison_elementwise_binary< ComparisonOperation::Less > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_fp32_comparison_elementwise_binary< ComparisonOperation::LessEqual > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template<ArithmeticOperation op, typename VectorType >
VectorType::type	elementwise_arithm_op (const typename VectorType::type &a, const typename VectorType::type &b)
template<ArithmeticOperation op, typename ScalarType , typename VectorType >
VectorType::type	elementwise_arithm_op_broadcast (const typename VectorType::type &a, const ScalarType &broadcast_value, const bool reorder)
template<typename InputScalarType , typename OutputScalarType , typename InputVectorType >
void	elementwise_op (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window, OutputScalarType(*scalar_func)(const InputScalarType &, const InputScalarType &), int(*broadcast_func)(int, int, int, const InputScalarType *, const InputScalarType &, OutputScalarType *, const bool), int(*neon_func)(int, int, int, const InputScalarType *, const InputScalarType *, OutputScalarType *))
template<ArithmeticOperation op, typename ScalarType >
ScalarType	elementwise_arithm_op_scalar (const ScalarType &a, const ScalarType &b)
template<>
int32x4_t	elementwise_arithm_op< ArithmeticOperation::DIV, typename wrapper::traits::neon_vector< int32_t, 4 > > (const int32x4_t &a, const int32x4_t &b)
template<>
float32x4_t	elementwise_arithm_op< ArithmeticOperation::DIV, typename wrapper::traits::neon_vector< float, 4 > > (const float32x4_t &a, const float32x4_t &b)
template<>
float32x4_t	elementwise_arithm_op< ArithmeticOperation::POWER, typename wrapper::traits::neon_vector< float, 4 > > (const float32x4_t &a, const float32x4_t &b)
template<ArithmeticOperation op, typename ScalarType , typename VectorType >
int	elementwise_arithm_op_loop (int window_start_x, int window_end_x, int window_step_x, const ScalarType *input1_ptr, const ScalarType *input2_ptr, ScalarType *output_ptr)
template<ArithmeticOperation op, typename ScalarType , typename VectorType >
int	elementwise_arithm_op_broadcast_loop (int window_start_x, int window_end_x, int window_step_x, const ScalarType *non_broadcast_input_ptr, const ScalarType &broadcast_value, ScalarType *output_ptr, const bool reorder)
template<ArithmeticOperation op, typename VectorType >
void	elementwise_arithm_op (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template<ComparisonOperation op, typename InputScalarType >
uint8_t	elementwise_comp_op_scalar (const InputScalarType &a, const InputScalarType &b)
template<ComparisonOperation op, typename InputVectorType , typename OutputVectorType >
OutputVectorType	elementwise_comp_op (const InputVectorType &a, const InputVectorType &b)
template<ComparisonOperation op, typename InputScalarType , typename InputVectorType , typename OutputVectorType >
OutputVectorType	elementwise_comp_op_broadcast (const InputVectorType &a, const InputScalarType &broadcast_value, const bool reorder)
template<ComparisonOperation op, typename InputScalarType , typename InputVectorType >
int	elementwise_comp_op_broadcast_8_loop (int window_start_x, int window_end_x, int window_step_x, const InputScalarType *non_broadcast_input_ptr, const InputScalarType &broadcast_value, uint8_t *output_ptr, const bool reorder)
template<ComparisonOperation op, typename InputScalarType , typename InputVectorType >
int	elementwise_comp_op_broadcast_16_loop (int window_start_x, int window_end_x, int window_step_x, const InputScalarType *non_broadcast_input_ptr, const InputScalarType &broadcast_value, uint8_t *output_ptr, const bool reorder)
template<ComparisonOperation op, typename InputScalarType , typename InputVectorType >
int	elementwise_comp_op_broadcast_32_loop (int window_start_x, int window_end_x, int window_step_x, const InputScalarType *non_broadcast_input_ptr, const InputScalarType &broadcast_value, uint8_t *output_ptr, const bool reorder)
template<ComparisonOperation op, typename InputScalarType , typename InputVectorType >
int	elementwise_comp_op_8_loop (int window_start_x, int window_end_x, int window_step_x, const InputScalarType *input1_ptr, const InputScalarType *input2_ptr, uint8_t *output_ptr)
template<ComparisonOperation op, typename InputScalarType , typename InputVectorType >
int	elementwise_comp_op_16_loop (int window_start_x, int window_end_x, int window_step_x, const InputScalarType *input1_ptr, const InputScalarType *input2_ptr, uint8_t *output_ptr)
template<ComparisonOperation op, typename InputScalarType , typename InputVectorType >
int	elementwise_comp_op_32_loop (int window_start_x, int window_end_x, int window_step_x, const InputScalarType *input1_ptr, const InputScalarType *input2_ptr, uint8_t *output_ptr)
template<ComparisonOperation op, typename InputScalarType , typename InputVectorType >
void	elementwise_comp_op_8 (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template<ComparisonOperation op, typename InputScalarType , typename InputVectorType >
void	elementwise_comp_op_16 (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template<ComparisonOperation op, typename InputScalarType , typename InputVectorType >
void	elementwise_comp_op_32 (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
float32x4x4_t	load_quantized (const uint8_t *input1_ptr, const int32x4_t &offset, const float32x4_t &scale)
float32x4x4_t	load_quantized_signed (const int8_t *input1_ptr, const int32x4_t &offset, const float32x4_t &scale)
void	store_quantized (uint8_t *output_ptr, const uint32x4x4_t &out)
void	store_quantized (uint8_t *output_ptr, const int32x4x4_t &out)
void	store_quantized (uint8_t *output_ptr, const float32x4x4_t &rf, const float32x4_t &offset, const float32x4_t &invscale)
void	store_quantized_signed (int8_t *output_ptr, const int32x4x4_t &out)
void	store_quantized_signed (int8_t *output_ptr, const float32x4x4_t &rf, const float32x4_t &offset, const float32x4_t &invscale)
template<ArithmeticOperation op>
uint8_t	elementwise_arithm_op_quantized_scalar (const float &a, const float &b, UniformQuantizationInfo qinfo)
template<ArithmeticOperation op>
int8_t	elementwise_arithm_op_quantized_signed_scalar (const float &a, const float &b, UniformQuantizationInfo qinfo)
template<ArithmeticOperation op>
float32x4x4_t	elementwise_arithm_op (const float32x4x4_t &a, const float32x4x4_t &b)
template<ComparisonOperation op>
uint8_t	elementwise_comp_op_quantized_scalar (const float &a, const float &b, UniformQuantizationInfo qinfo)
template<ComparisonOperation op>
uint32x4x4_t	elementwise_comp_op (const float32x4x4_t &a, const float32x4x4_t &b)
template<ArithmeticOperation op>
int	elementwise_arithm_op_quantized_loop (int window_start_x, int window_end_x, int window_step_x, const uint8_t *input1_ptr, const uint8_t *input2_ptr, uint8_t *output_ptr, int32x4_t voffset1, int32x4_t voffset2, float32x4_t vscale1, float32x4_t vscale2, float32x4_t voffseto, float32x4_t invvscaleo)
template<ArithmeticOperation op>
int	elementwise_arithm_op_quantized_singed_loop (int window_start_x, int window_end_x, int window_step_x, const int8_t *input1_ptr, const int8_t *input2_ptr, int8_t *output_ptr, int32x4_t voffset1, int32x4_t voffset2, float32x4_t vscale1, float32x4_t vscale2, float32x4_t voffseto, float32x4_t invvscaleo)
template<ArithmeticOperation op>
int	elementwise_arithm_op_quantized_broadcast_loop (int window_start_x, int window_end_x, int window_step_x, const uint8_t *non_broadcast_input_ptr, float32x4x4_t broadcast_vector, uint8_t *output_ptr, int32x4_t voffset_non_broadcast, float32x4_t vscale_non_broadcast, float32x4_t voffseto, float32x4_t invvscaleo, bool reorder)
template<ArithmeticOperation op>
int	elementwise_arithm_op_quantized_signed_broadcast_loop (int window_start_x, int window_end_x, int window_step_x, const int8_t *non_broadcast_input_ptr, float32x4x4_t broadcast_vector, int8_t *output_ptr, int32x4_t voffset_non_broadcast, float32x4_t vscale_non_broadcast, float32x4_t voffseto, float32x4_t invvscaleo, bool reorder)
template<ComparisonOperation op>
int	elementwise_comp_op_quantized_loop (int window_start_x, int window_end_x, int window_step_x, const uint8_t *input1_ptr, const uint8_t *input2_ptr, uint8_t *output_ptr, int32x4_t voffset1, int32x4_t voffset2, float32x4_t vscale1, float32x4_t vscale2, float32x4_t voffseto, float32x4_t invvscaleo)
template<ComparisonOperation op>
int	elementwise_comp_op_quantized_signed_loop (int window_start_x, int window_end_x, int window_step_x, const int8_t *input1_ptr, const int8_t *input2_ptr, uint8_t *output_ptr, int32x4_t voffset1, int32x4_t voffset2, float32x4_t vscale1, float32x4_t vscale2, float32x4_t voffseto, float32x4_t invvscaleo)
template<ComparisonOperation op>
int	elementwise_comp_op_quantized_broadcast_loop (int window_start_x, int window_end_x, int window_step_x, const uint8_t *non_broadcast_input_ptr, float32x4x4_t broadcast_vector, uint8_t *output_ptr, int32x4_t voffset_non_broadcast, float32x4_t vscale_non_broadcast, float32x4_t voffseto, float32x4_t invvscaleo, bool reorder)
template<ComparisonOperation op>
int	elementwise_comp_op_quantized_signed_broadcast_loop (int window_start_x, int window_end_x, int window_step_x, const int8_t *non_broadcast_input_ptr, float32x4x4_t broadcast_vector, uint8_t *output_ptr, int32x4_t voffset_non_broadcast, float32x4_t vscale_non_broadcast, float32x4_t voffseto, float32x4_t invvscaleo, bool reorder)
void	elementwise_op_quantized (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window, uint8_t(*scalar_func)(const float &, const float &, UniformQuantizationInfo), int(*broadcast_func)(int, int, int, const uint8_t *, float32x4x4_t, uint8_t *, int32x4_t, float32x4_t, float32x4_t, float32x4_t, const bool), int(*neon_func)(int, int, int, const uint8_t *, const uint8_t *, uint8_t *, int32x4_t, int32x4_t, float32x4_t, float32x4_t, float32x4_t, float32x4_t))
void	elementwise_comp_quantized_signed (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window, uint8_t(*scalar_func)(const float &, const float &, UniformQuantizationInfo), int(*broadcast_func)(int, int, int, const int8_t *, float32x4x4_t, uint8_t *, int32x4_t, float32x4_t, float32x4_t, float32x4_t, const bool), int(*neon_func)(int, int, int, const int8_t *, const int8_t *, uint8_t *, int32x4_t, int32x4_t, float32x4_t, float32x4_t, float32x4_t, float32x4_t))
void	elementwise_op_quantized_signed (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window, int8_t(*scalar_func)(const float &, const float &, UniformQuantizationInfo), int(*broadcast_func)(int, int, int, const int8_t *, float32x4x4_t, int8_t *, int32x4_t, float32x4_t, float32x4_t, float32x4_t, const bool), int(*neon_func)(int, int, int, const int8_t *, const int8_t *, int8_t *, int32x4_t, int32x4_t, float32x4_t, float32x4_t, float32x4_t, float32x4_t))
template<ArithmeticOperation op>
void	elementwise_arithm_op_quantized (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template<ArithmeticOperation op>
void	elementwise_arithm_op_quantized_signed (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template<ComparisonOperation op>
void	elementwise_comp_op_quantized (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template<ComparisonOperation op>
void	elementwise_comp_op_quantized_signed (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template<ArithmeticOperation op>
void	neon_s32_elementwise_binary (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_s32_elementwise_binary< ArithmeticOperation::ADD > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_s32_elementwise_binary< ArithmeticOperation::SUB > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_s32_elementwise_binary< ArithmeticOperation::DIV > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_s32_elementwise_binary< ArithmeticOperation::MIN > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_s32_elementwise_binary< ArithmeticOperation::MAX > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_s32_elementwise_binary< ArithmeticOperation::SQUARED_DIFF > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_s32_elementwise_binary< ArithmeticOperation::POWER > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_s32_elementwise_binary< ArithmeticOperation::PRELU > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template<ArithmeticOperation op>
void	neon_s16_elementwise_binary (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_s16_elementwise_binary< ArithmeticOperation::ADD > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_s16_elementwise_binary< ArithmeticOperation::SUB > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_s16_elementwise_binary< ArithmeticOperation::DIV > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_s16_elementwise_binary< ArithmeticOperation::MIN > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_s16_elementwise_binary< ArithmeticOperation::MAX > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_s16_elementwise_binary< ArithmeticOperation::SQUARED_DIFF > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_s16_elementwise_binary< ArithmeticOperation::POWER > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_s16_elementwise_binary< ArithmeticOperation::PRELU > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template<ComparisonOperation op>
void	neon_u8_comparison_elementwise_binary (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_u8_comparison_elementwise_binary< ComparisonOperation::Equal > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_u8_comparison_elementwise_binary< ComparisonOperation::NotEqual > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_u8_comparison_elementwise_binary< ComparisonOperation::Greater > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_u8_comparison_elementwise_binary< ComparisonOperation::GreaterEqual > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_u8_comparison_elementwise_binary< ComparisonOperation::Less > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_u8_comparison_elementwise_binary< ComparisonOperation::LessEqual > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template<ComparisonOperation op>
void	neon_s16_comparison_elementwise_binary (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_s16_comparison_elementwise_binary< ComparisonOperation::Equal > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_s16_comparison_elementwise_binary< ComparisonOperation::NotEqual > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_s16_comparison_elementwise_binary< ComparisonOperation::Greater > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_s16_comparison_elementwise_binary< ComparisonOperation::GreaterEqual > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_s16_comparison_elementwise_binary< ComparisonOperation::Less > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_s16_comparison_elementwise_binary< ComparisonOperation::LessEqual > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template<ComparisonOperation op>
void	neon_s32_comparison_elementwise_binary (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_s32_comparison_elementwise_binary< ComparisonOperation::Equal > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_s32_comparison_elementwise_binary< ComparisonOperation::NotEqual > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_s32_comparison_elementwise_binary< ComparisonOperation::Greater > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_s32_comparison_elementwise_binary< ComparisonOperation::GreaterEqual > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_s32_comparison_elementwise_binary< ComparisonOperation::Less > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_s32_comparison_elementwise_binary< ComparisonOperation::LessEqual > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template<ArithmeticOperation op>
void	neon_qasymm8_elementwise_binary (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_qasymm8_elementwise_binary< ArithmeticOperation::ADD > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_qasymm8_elementwise_binary< ArithmeticOperation::SUB > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_qasymm8_elementwise_binary< ArithmeticOperation::DIV > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_qasymm8_elementwise_binary< ArithmeticOperation::MIN > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_qasymm8_elementwise_binary< ArithmeticOperation::MAX > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_qasymm8_elementwise_binary< ArithmeticOperation::SQUARED_DIFF > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_qasymm8_elementwise_binary< ArithmeticOperation::POWER > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_qasymm8_elementwise_binary< ArithmeticOperation::PRELU > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template<ComparisonOperation op>
void	neon_qasymm8_comparison_elementwise_binary (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_qasymm8_comparison_elementwise_binary< ComparisonOperation::Equal > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_qasymm8_comparison_elementwise_binary< ComparisonOperation::NotEqual > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_qasymm8_comparison_elementwise_binary< ComparisonOperation::Greater > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_qasymm8_comparison_elementwise_binary< ComparisonOperation::GreaterEqual > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_qasymm8_comparison_elementwise_binary< ComparisonOperation::Less > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_qasymm8_comparison_elementwise_binary< ComparisonOperation::LessEqual > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template<ArithmeticOperation op>
void	neon_qasymm8_signed_elementwise_binary (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_qasymm8_signed_elementwise_binary< ArithmeticOperation::ADD > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_qasymm8_signed_elementwise_binary< ArithmeticOperation::SUB > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_qasymm8_signed_elementwise_binary< ArithmeticOperation::DIV > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_qasymm8_signed_elementwise_binary< ArithmeticOperation::MIN > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_qasymm8_signed_elementwise_binary< ArithmeticOperation::MAX > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_qasymm8_signed_elementwise_binary< ArithmeticOperation::SQUARED_DIFF > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_qasymm8_signed_elementwise_binary< ArithmeticOperation::POWER > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_qasymm8_signed_elementwise_binary< ArithmeticOperation::PRELU > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template<ComparisonOperation op>
void	neon_qasymm8_signed_comparison_elementwise_binary (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_qasymm8_signed_comparison_elementwise_binary< ComparisonOperation::Equal > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_qasymm8_signed_comparison_elementwise_binary< ComparisonOperation::NotEqual > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_qasymm8_signed_comparison_elementwise_binary< ComparisonOperation::Greater > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_qasymm8_signed_comparison_elementwise_binary< ComparisonOperation::GreaterEqual > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_qasymm8_signed_comparison_elementwise_binary< ComparisonOperation::Less > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	neon_qasymm8_signed_comparison_elementwise_binary< ComparisonOperation::LessEqual > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template<ArithmeticOperation op>
void	sve_fp32_elementwise_binary (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_fp32_elementwise_binary< ArithmeticOperation::ADD > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_fp32_elementwise_binary< ArithmeticOperation::SUB > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_fp32_elementwise_binary< ArithmeticOperation::DIV > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_fp32_elementwise_binary< ArithmeticOperation::MIN > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_fp32_elementwise_binary< ArithmeticOperation::MAX > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_fp32_elementwise_binary< ArithmeticOperation::SQUARED_DIFF > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_fp32_elementwise_binary< ArithmeticOperation::POWER > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_fp32_elementwise_binary< ArithmeticOperation::PRELU > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template<ComparisonOperation op>
void	sve_fp32_comparison_elementwise_binary (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_fp32_comparison_elementwise_binary< ComparisonOperation::Equal > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_fp32_comparison_elementwise_binary< ComparisonOperation::NotEqual > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_fp32_comparison_elementwise_binary< ComparisonOperation::Greater > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_fp32_comparison_elementwise_binary< ComparisonOperation::GreaterEqual > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_fp32_comparison_elementwise_binary< ComparisonOperation::Less > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_fp32_comparison_elementwise_binary< ComparisonOperation::LessEqual > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template<typename ScalarType >
void	elementwise_arithmetic_op (const ITensor *in1, const ITensor *in2, ITensor *out, ArithmeticOperation op, const Window &window)
template void	elementwise_arithmetic_op< float32_t > (const ITensor *in1, const ITensor *in2, ITensor *out, const ArithmeticOperation op, const Window &window)
template void	elementwise_arithmetic_op< float16_t > (const ITensor *in1, const ITensor *in2, ITensor *out, const ArithmeticOperation op, const Window &window)
template void	elementwise_arithmetic_op< int16_t > (const ITensor *in1, const ITensor *in2, ITensor *out, const ArithmeticOperation op, const Window &window)
template void	elementwise_arithmetic_op< int32_t > (const ITensor *in1, const ITensor *in2, ITensor *out, const ArithmeticOperation op, const Window &window)
template<typename InputScalarType , typename OutputScalarType >
void	elementwise_comparison_op (const ITensor *in1, const ITensor *in2, ITensor *out, ComparisonOperation op, const Window &window)
template void	elementwise_comparison_op< float32_t > (const ITensor *in1, const ITensor *in2, ITensor *out, const ComparisonOperation op, const Window &window)
template void	elementwise_comparison_op< float16_t > (const ITensor *in1, const ITensor *in2, ITensor *out, const ComparisonOperation op, const Window &window)
template void	elementwise_comparison_op< uint8_t > (const ITensor *in1, const ITensor *in2, ITensor *out, const ComparisonOperation op, const Window &window)
template void	elementwise_comparison_op< int16_t > (const ITensor *in1, const ITensor *in2, ITensor *out, const ComparisonOperation op, const Window &window)
template void	elementwise_comparison_op< int32_t > (const ITensor *in1, const ITensor *in2, ITensor *out, const ComparisonOperation op, const Window &window)
template<>
svint32_t	elementwise_pow< svint32_t > (svbool_t &pg, const svint32_t &a, const svint32_t &b)
template<>
svint32_t	elementwise_div< svint32_t > (svbool_t &pg, const svint32_t &a, const svint32_t &b)
template<>
svint16_t	elementwise_div< svint16_t > (svbool_t &pg, const svint16_t &a, const svint16_t &b)
template<typename VectorType >
VectorType	elementwise_pow (svbool_t &pg, const VectorType &a, const VectorType &b)
template<typename VectorType >
VectorType	elementwise_div (svbool_t &pg, const VectorType &a, const VectorType &b)
template<uint32_t bytewidth>
svbool_t	narrow_to_byte_predicate (svbool_t pg)
template<typename VectorType >
VectorType	elementwise_arithmetic_op (svbool_t &pg, const VectorType &a, const VectorType &b, ArithmeticOperation op)
template<typename InputVectorType , typename OutputVectorType >
OutputVectorType	elementwise_comparison_op (svbool_t &pg, const InputVectorType &a, const InputVectorType &b, ComparisonOperation op)
template<ArithmeticOperation op>
void	sve_s32_elementwise_binary (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_s32_elementwise_binary< ArithmeticOperation::ADD > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_s32_elementwise_binary< ArithmeticOperation::SUB > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_s32_elementwise_binary< ArithmeticOperation::DIV > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_s32_elementwise_binary< ArithmeticOperation::MIN > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_s32_elementwise_binary< ArithmeticOperation::MAX > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_s32_elementwise_binary< ArithmeticOperation::SQUARED_DIFF > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_s32_elementwise_binary< ArithmeticOperation::POWER > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_s32_elementwise_binary< ArithmeticOperation::PRELU > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template<ArithmeticOperation op>
void	sve_s16_elementwise_binary (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_s16_elementwise_binary< ArithmeticOperation::ADD > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_s16_elementwise_binary< ArithmeticOperation::SUB > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_s16_elementwise_binary< ArithmeticOperation::DIV > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_s16_elementwise_binary< ArithmeticOperation::MIN > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_s16_elementwise_binary< ArithmeticOperation::MAX > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_s16_elementwise_binary< ArithmeticOperation::SQUARED_DIFF > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_s16_elementwise_binary< ArithmeticOperation::POWER > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_s16_elementwise_binary< ArithmeticOperation::PRELU > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template<ComparisonOperation op>
void	sve_u8_comparison_elementwise_binary (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_u8_comparison_elementwise_binary< ComparisonOperation::Equal > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_u8_comparison_elementwise_binary< ComparisonOperation::NotEqual > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_u8_comparison_elementwise_binary< ComparisonOperation::Greater > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_u8_comparison_elementwise_binary< ComparisonOperation::GreaterEqual > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_u8_comparison_elementwise_binary< ComparisonOperation::Less > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_u8_comparison_elementwise_binary< ComparisonOperation::LessEqual > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template<ComparisonOperation op>
void	sve_s16_comparison_elementwise_binary (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_s16_comparison_elementwise_binary< ComparisonOperation::Equal > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_s16_comparison_elementwise_binary< ComparisonOperation::NotEqual > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_s16_comparison_elementwise_binary< ComparisonOperation::Greater > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_s16_comparison_elementwise_binary< ComparisonOperation::GreaterEqual > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_s16_comparison_elementwise_binary< ComparisonOperation::Less > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_s16_comparison_elementwise_binary< ComparisonOperation::LessEqual > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template<ComparisonOperation op>
void	sve_s32_comparison_elementwise_binary (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_s32_comparison_elementwise_binary< ComparisonOperation::Equal > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_s32_comparison_elementwise_binary< ComparisonOperation::NotEqual > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_s32_comparison_elementwise_binary< ComparisonOperation::Greater > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_s32_comparison_elementwise_binary< ComparisonOperation::GreaterEqual > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_s32_comparison_elementwise_binary< ComparisonOperation::Less > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve_s32_comparison_elementwise_binary< ComparisonOperation::LessEqual > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
svfloat32x4_t	load_quantized (const int8_t *ptr, svbool_t pg, const svint32_t &offset, const svfloat32_t &scale)
svfloat32x4_t	load_quantized (const uint8_t *ptr, svbool_t pg, const svint32_t &offset, const svfloat32_t &scale)
void	store_quantized (uint8_t *ptr, svbool_t pg, svfloat32x4_t data, const svint32_t &offset, const svfloat32_t &inv_scale)
void	store_quantized (int8_t *ptr, svbool_t pg, svfloat32x4_t data, const svint32_t &offset, const svfloat32_t &inv_scale)
template<typename ScalarType >
void	elementwise_arithmetic_quantized_op (const ITensor *in1, const ITensor *in2, ITensor *out, ArithmeticOperation op, const Window &window)
template<typename InputScalarType , typename OutputScalarType = uint8_t>
void	elementwise_comparison_quantized_op (const ITensor *in1, const ITensor *in2, ITensor *out, ComparisonOperation op, const Window &window)
template<ArithmeticOperation op>
void	sve2_qasymm8_elementwise_binary (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve2_qasymm8_elementwise_binary< ArithmeticOperation::ADD > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve2_qasymm8_elementwise_binary< ArithmeticOperation::SUB > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve2_qasymm8_elementwise_binary< ArithmeticOperation::DIV > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve2_qasymm8_elementwise_binary< ArithmeticOperation::MIN > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve2_qasymm8_elementwise_binary< ArithmeticOperation::MAX > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve2_qasymm8_elementwise_binary< ArithmeticOperation::SQUARED_DIFF > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve2_qasymm8_elementwise_binary< ArithmeticOperation::POWER > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve2_qasymm8_elementwise_binary< ArithmeticOperation::PRELU > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template<ComparisonOperation op>
void	sve2_qasymm8_comparison_elementwise_binary (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve2_qasymm8_comparison_elementwise_binary< ComparisonOperation::Equal > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve2_qasymm8_comparison_elementwise_binary< ComparisonOperation::NotEqual > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve2_qasymm8_comparison_elementwise_binary< ComparisonOperation::Greater > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve2_qasymm8_comparison_elementwise_binary< ComparisonOperation::GreaterEqual > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve2_qasymm8_comparison_elementwise_binary< ComparisonOperation::Less > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve2_qasymm8_comparison_elementwise_binary< ComparisonOperation::LessEqual > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template<ArithmeticOperation op>
void	sve2_qasymm8_signed_elementwise_binary (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve2_qasymm8_signed_elementwise_binary< ArithmeticOperation::ADD > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve2_qasymm8_signed_elementwise_binary< ArithmeticOperation::SUB > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve2_qasymm8_signed_elementwise_binary< ArithmeticOperation::DIV > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve2_qasymm8_signed_elementwise_binary< ArithmeticOperation::MIN > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve2_qasymm8_signed_elementwise_binary< ArithmeticOperation::MAX > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve2_qasymm8_signed_elementwise_binary< ArithmeticOperation::SQUARED_DIFF > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve2_qasymm8_signed_elementwise_binary< ArithmeticOperation::POWER > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve2_qasymm8_signed_elementwise_binary< ArithmeticOperation::PRELU > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template<ComparisonOperation op>
void	sve2_qasymm8_signed_comparison_elementwise_binary (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve2_qasymm8_signed_comparison_elementwise_binary< ComparisonOperation::Equal > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve2_qasymm8_signed_comparison_elementwise_binary< ComparisonOperation::NotEqual > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve2_qasymm8_signed_comparison_elementwise_binary< ComparisonOperation::Greater > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve2_qasymm8_signed_comparison_elementwise_binary< ComparisonOperation::GreaterEqual > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve2_qasymm8_signed_comparison_elementwise_binary< ComparisonOperation::Less > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	sve2_qasymm8_signed_comparison_elementwise_binary< ComparisonOperation::LessEqual > (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template<ArithmeticOperation op>
void	sve_fp16_elementwise_binary (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template<ArithmeticOperation op>
void	neon_fp16_elementwise_binary (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template<ComparisonOperation op>
void	sve_fp16_comparison_elementwise_binary (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template<ComparisonOperation op>
void	neon_fp16_comparison_elementwise_binary (const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
void	neon_fp32_elementwise_unary (const ITensor *in, ITensor *out, const Window &window, ElementWiseUnary op)
template<typename ScalarType >
ScalarType	elementwise_op_scalar_imp (ElementWiseUnary op, const ScalarType &a)
template<typename ScalarType , typename VectorType >
VectorType	elementwise_op_imp (ElementWiseUnary op, const VectorType &a)
template<typename ScalarType >
void	elementwise_op (const ITensor *in, ITensor *out, const Window &window, ElementWiseUnary op)
template void	elementwise_op< float > (const ITensor *in, ITensor *out, const Window &window, ElementWiseUnary op)
template void	elementwise_op< int32_t > (const ITensor *in, ITensor *out, const Window &window, ElementWiseUnary op)
void	neon_s32_elementwise_unary (const ITensor *in, ITensor *out, const Window &window, ElementWiseUnary op)
void	sve_fp32_elementwise_unary (const ITensor *in, ITensor *out, const Window &window, ElementWiseUnary op)
template<typename ScalarType , typename VectorType >
std::enable_if< utils::traits::is_floating_point< ScalarType >::value, VectorType >::type	elementwise_op_sve_imp (svbool_t pg, ElementWiseUnary op, const VectorType &a)
template<typename ScalarType , typename VectorType >
std::enable_if< std::is_integral< ScalarType >::value, VectorType >::type	elementwise_op_sve_imp (svbool_t pg, ElementWiseUnary op, const VectorType &a)
template<typename ScalarType >
void	elementwise_sve_op (const ITensor *in, ITensor *out, const Window &window, ElementWiseUnary op)
template void	elementwise_sve_op< float16_t > (const ITensor *in, ITensor *out, const Window &window, ElementWiseUnary op)
template void	elementwise_sve_op< float32_t > (const ITensor *in, ITensor *out, const Window &window, ElementWiseUnary op)
template void	elementwise_sve_op< int32_t > (const ITensor *in, ITensor *out, const Window &window, ElementWiseUnary op)
void	sve_s32_elementwise_unary (const ITensor *in, ITensor *out, const Window &window, ElementWiseUnary op)
void	sve_fp16_elementwise_unary (const ITensor *in, ITensor *out, const Window &window, ElementWiseUnary op)
void	neon_fp16_elementwise_unary (const ITensor *in, ITensor *out, const Window &window, ElementWiseUnary op)
void	fp16_neon_floor (const void *src, void *dst, int len)
void	fp32_neon_floor (const void *src, void *dst, int len)
void	fused_batch_normalization_conv_f32 (const ITensor *conv_weights, const ITensor *conv_bias, ITensor *fused_weights, ITensor *fused_bias, const ITensor *bn_mean, const ITensor *bn_var, const ITensor *bn_beta, const ITensor *bn_gamma, float epsilon, const Window &window)
template<typename T >
void	fused_batch_normalization_conv (const ITensor *conv_weights, const ITensor *conv_bias, ITensor *fused_weights, ITensor *fused_bias, const ITensor *bn_mean, const ITensor *bn_var, const ITensor *bn_beta, const ITensor *bn_gamma, float epsilon, const Window &window)
template void	fused_batch_normalization_conv< float32_t > (const ITensor *conv_weights, const ITensor *conv_bias, ITensor *fused_weights, ITensor *fused_bias, const ITensor *bn_mean, const ITensor *bn_var, const ITensor *bn_beta, const ITensor *bn_gamma, float epsilon, const Window &window)
void	fused_batch_normalization_conv_f16 (const ITensor *conv_weights, const ITensor *conv_bias, ITensor *fused_weights, ITensor *fused_bias, const ITensor *bn_mean, const ITensor *bn_var, const ITensor *bn_beta, const ITensor *bn_gamma, float epsilon, const Window &window)
void	fused_batch_normalization_dwc_nhwc_f16 (const ITensor *dwc_weights, const ITensor *dwc_bias, ITensor *fused_weights, ITensor *fused_bias, const ITensor *bn_mean, const ITensor *bn_var, const ITensor *bn_beta, const ITensor *bn_gamma, float epsilon, const Window &window)
void	fused_batch_normalization_dwc_nhwc_f32 (const ITensor *dwc_weights, const ITensor *dwc_bias, ITensor *fused_weights, ITensor *fused_bias, const ITensor *bn_mean, const ITensor *bn_var, const ITensor *bn_beta, const ITensor *bn_gamma, float epsilon, const Window &window)
void	fused_batch_normalization_dwc_nchw_f16 (const ITensor *dwc_weights, const ITensor *dwc_bias, ITensor *fused_weights, ITensor *fused_bias, const ITensor *bn_mean, const ITensor *bn_var, const ITensor *bn_beta, const ITensor *bn_gamma, float epsilon, const Window &window)
void	fused_batch_normalization_dwc_nchw_f32 (const ITensor *dwc_weights, const ITensor *dwc_bias, ITensor *fused_weights, ITensor *fused_bias, const ITensor *bn_mean, const ITensor *bn_var, const ITensor *bn_beta, const ITensor *bn_gamma, float epsilon, const Window &window)
template<typename T >
void	fused_batch_normalization_dwc_nchw (const ITensor *dwc_weights, const ITensor *dwc_bias, ITensor *fused_weights, ITensor *fused_bias, const ITensor *bn_mean, const ITensor *bn_var, const ITensor *bn_beta, const ITensor *bn_gamma, float epsilon, const Window &window)
template<typename T >
void	fused_batch_normalization_dwc_nhwc (const ITensor *dwc_weights, const ITensor *dwc_bias, ITensor *fused_weights, ITensor *fused_bias, const ITensor *bn_mean, const ITensor *bn_var, const ITensor *bn_beta, const ITensor *bn_gamma, float epsilon, const Window &window)
template void	fused_batch_normalization_dwc_nhwc< float32_t > (const ITensor *dwc_weights, const ITensor *dwc_bias, ITensor *fused_weights, ITensor *fused_bias, const ITensor *bn_mean, const ITensor *bn_var, const ITensor *bn_beta, const ITensor *bn_gamma, float epsilon, const Window &window)
void	neon_fp32_gemm_matrix_add (const ITensor *src, ITensor *dst, const Window &window, float beta)
void	matrix_addition_f32 (const ITensor *src, ITensor *dst, const Window &window, float beta)
void	neon_fp16_gemm_matrix_add (const ITensor *src, ITensor *dst, const Window &window, float beta)
void	neon_fp32_gemm_matrix_mul (const ITensor *lhs, const ITensor *rhs, ITensor *dst, const Window &window, const ThreadInfo &info, float alpha, const bool is_dst_vector)
void	vector_matrix_multiply_f32 (const ITensor *lhs, const ITensor *rhs, ITensor *dst, const Window &window, const ThreadInfo &info, float alpha)
void	matrix_matrix_multiply_f32 (const ITensor *lhs, const ITensor *rhs, ITensor *dst, const Window &window, const ThreadInfo &info, float alpha)
void	neon_fp16_gemm_matrix_mul (const ITensor *lhs, const ITensor *rhs, ITensor *dst, const Window &window, const ThreadInfo &info, float alpha, const bool is_dst_vector)
void	neon_fp32_computeallanchors (const ITensor *anchors, ITensor *all_anchors, ComputeAnchorsInfo anchors_info, const Window &window)
template<typename T >
void	compute_all_anchors (const ITensor *anchors, ITensor *all_anchors, ComputeAnchorsInfo anchors_info, const Window &window)
template void	compute_all_anchors< float > (const ITensor *anchors, ITensor *all_anchors, ComputeAnchorsInfo anchors_info, const Window &window)
void	compute_all_anchors_qasymm16 (const ITensor *anchors, ITensor *all_anchors, ComputeAnchorsInfo anchors_info, const Window &window)
void	neon_qu16_computeallanchors (const ITensor *anchors, ITensor *all_anchors, ComputeAnchorsInfo anchors_info, const Window &window)
void	neon_fp16_computeallanchors (const ITensor *anchors, ITensor *all_anchors, ComputeAnchorsInfo anchors_info, const Window &window)
void	neon_fp32_instancenorm (ITensor *input, ITensor *output, float gamma, float beta, float epsilon, bool use_mixed_precision, const Window &window)
template<typename InputType , typename AccType >
void	vector_float_sum (AccType &result, AccType &result_square, const InputType &inputs)
template<typename InputType , typename AccType >
InputType	vector_float_norm (const InputType &inputs, const AccType &vec_mean, const AccType &vec_multip, const AccType &vec_beta)
template<typename T , typename AccType >
void	instance_normalization_nchw (ITensor *input, ITensor *output, float gamma, float beta, float epsilon, const Window &window)
template void	instance_normalization_nchw< float > (ITensor *input, ITensor *output, float gamma, float beta, float epsilon, const Window &window)
void	neon_fp16_instancenorm (ITensor *input, ITensor *output, float gamma, float beta, float epsilon, bool use_mixed_precision, const Window &window)
void	neon_fp32_l2_normalize_x (const ITensor *in, const ITensor *sum, ITensor *out, float epsilon, const Window &window, size_t unused_axis)
void	neon_fp32_l2_normalize_yz (const ITensor *in, const ITensor *sum, ITensor *out, float epsilon, const Window &window, size_t axis)
template<typename T , int S>
void	l2_normalize_x (const ITensor *in, const ITensor *sum, ITensor *out, float epsilon, const Window &window)
template<typename T , int S>
void	l2_normalize_yz (const ITensor *in, const ITensor *sum, ITensor *out, float epsilon, const Window &window, size_t axis)
template void	l2_normalize_yz< float, 4 > (const ITensor *in, const ITensor *sum, ITensor *out, float epsilon, const Window &window, size_t axis)
template void	l2_normalize_x< float, 4 > (const ITensor *in, const ITensor *sum, ITensor *out, float epsilon, const Window &window)
void	neon_fp16_l2_normalize_x (const ITensor *in, const ITensor *sum, ITensor *out, float epsilon, const Window &window, size_t axis)
void	neon_fp16_l2_normalize_yz (const ITensor *in, const ITensor *sum, ITensor *out, float epsilon, const Window &window, size_t axis)
void	neon_fp32_maxunpooling (const ITensor *input, const ITensor *indices, ITensor *output, const Window &window)
template<typename T >
void	max_unpooling (const ITensor *input, const ITensor *indices, ITensor *output, const Window &window)
template void	max_unpooling< float > (const ITensor *input, const ITensor *indices, ITensor *output, const Window &window)
template void	max_unpooling< int8_t > (const ITensor *input, const ITensor *indices, ITensor *output, const Window &window)
template void	max_unpooling< uint8_t > (const ITensor *input, const ITensor *indices, ITensor *output, const Window &window)
void	neon_qs8_maxunpooling (const ITensor *input, const ITensor *indices, ITensor *output, const Window &window)
void	neon_qu8_maxunpooling (const ITensor *input, const ITensor *indices, ITensor *output, const Window &window)
void	neon_fp16_maxunpooling (const ITensor *input, const ITensor *indices, ITensor *output, const Window &window)
void	neon_fp32_meanstddevnorm (ITensor *input, ITensor *output, float epsilon, const Window &window)
template<typename ScalarType , int size>
void	mean_stddev_normalization (ITensor *input, ITensor *output, float epsilon, const Window &window)
template void	mean_stddev_normalization< float, 4 > (ITensor *input, ITensor *output, float epsilon, const Window &window)
void	neon_fp16_meanstddevnorm (ITensor *input, ITensor *output, float epsilon, const Window &window)
void	poolingMxN_fp32_neon_nhwc (const ITensor *src, ITensor *dst0, ITensor *dst1, PoolingLayerInfo &pool_info, const Window &window_src, const Window &window)
void	poolingMxN_qasymm8_neon_nhwc (const ITensor *src0, ITensor *dst0, ITensor *dst1, PoolingLayerInfo &, const Window &window_src, const Window &window)
void	poolingMxN_qasymm8_signed_neon_nhwc (const ITensor *src0, ITensor *dst0, ITensor *dst1, PoolingLayerInfo &, const Window &window_src, const Window &window)
void	poolingMxN_fp16_neon_nhwc (const ITensor *src0, ITensor *dst0, ITensor *dst1, PoolingLayerInfo &, const Window &window_src, const Window &window)
template<typename T >
uint32_t	offset_no_padding (uint32_t padded_offset, const Coordinates &id, const ITensorInfo &info, int pool_stride_x, int pool_stride_y, DataLayout data_layout)
template<typename T >
void	poolingMxN_q8_neon_nhwc (const ITensor *src, ITensor *dst0, ITensor *dst1, PoolingLayerInfo &pool_info, const Window &window_src, const Window &window)
void	neon_q8_pool3d (const ITensor *src0, ITensor *dst0, Pooling3dLayerInfo &, const Window &window)
void	neon_q8_signed_pool3d (const ITensor *src0, ITensor *dst0, Pooling3dLayerInfo &, const Window &window)
void	neon_fp16_pool3d (const ITensor *src0, ITensor *dst0, Pooling3dLayerInfo &, const Window &window)
void	neon_fp32_pool3d (const ITensor *src0, ITensor *dst0, Pooling3dLayerInfo &, const Window &window)
template<typename T >
void	poolingMxNxD_fp_neon_ndhwc (const ITensor *src, ITensor *dst0, Pooling3dLayerInfo &pool_info, const Window &window)
template<typename T >
void	poolingMxNxD_q8_neon_ndhwc (const ITensor *src, ITensor *dst0, Pooling3dLayerInfo &pool_info, const Window &window)
template void	poolingMxNxD_fp_neon_ndhwc< float > (const ITensor *src, ITensor *dst0, Pooling3dLayerInfo &pool_info, const Window &window)
template void	poolingMxNxD_q8_neon_ndhwc< uint8_t > (const ITensor *src, ITensor *dst0, Pooling3dLayerInfo &pool_info, const Window &window)
template void	poolingMxNxD_q8_neon_ndhwc< int8_t > (const ITensor *src, ITensor *dst0, Pooling3dLayerInfo &pool_info, const Window &window)
template<typename T >
void	avg_poolingMxNxD_q8_neon_ndhwc (const ITensor *src, ITensor *dst0, Pooling3dLayerInfo &pool_info, const Window &window_out, const int window_step_x)
template<typename T >
void	max_poolingMxNxD_q8_neon_ndhwc (const ITensor *src, ITensor *dst0, Pooling3dLayerInfo &pool_info, const Window &window_out, const int window_step_x)
void	fp32_neon_range_function (ITensor *output, float start, float step, const Window &window)
template<typename T >
void	neon_range_function (ITensor *output, float start, float step, const Window &window)
template void	neon_range_function< uint8_t > (ITensor *output, float start, float step, const Window &window)
template void	neon_range_function< uint16_t > (ITensor *output, float start, float step, const Window &window)
template void	neon_range_function< uint32_t > (ITensor *output, float start, float step, const Window &window)
template void	neon_range_function< int8_t > (ITensor *output, float start, float step, const Window &window)
template void	neon_range_function< int16_t > (ITensor *output, float start, float step, const Window &window)
template void	neon_range_function< int32_t > (ITensor *output, float start, float step, const Window &window)
template void	neon_range_function< float32_t > (ITensor *output, float start, float step, const Window &window)
void	u8_neon_range_function (ITensor *output, float start, float step, const Window &window)
void	u16_neon_range_function (ITensor *output, float start, float step, const Window &window)
void	u32_neon_range_function (ITensor *output, float start, float step, const Window &window)
void	s8_neon_range_function (ITensor *output, float start, float step, const Window &window)
void	s16_neon_range_function (ITensor *output, float start, float step, const Window &window)
void	s32_neon_range_function (ITensor *output, float start, float step, const Window &window)
void	fp16_neon_range_function (ITensor *output, float start, float step, const Window &window)
void	neon_fp32_roialign (const ITensor *input, ITensor *output, const ITensor *rois, ROIPoolingLayerInfo pool_info, const Window &window, const ThreadInfo &info)
template<typename input_data_type >
input_data_type	roi_align_1x1 (const ITensor *input, unsigned int roi_batch, float region_start_x, float bin_size_x, int grid_size_x, float region_end_x, float region_start_y, float bin_size_y, int grid_size_y, float region_end_y, int pz)
	Average pooling over an aligned window. More...
template<typename input_data_type >
input_data_type	roi_align_1x1_qasymm8 (const ITensor *input, unsigned int roi_batch, float region_start_x, float bin_size_x, int grid_size_x, float region_end_x, float region_start_y, float bin_size_y, int grid_size_y, float region_end_y, int pz, const QuantizationInfo &out_qinfo)
	Average pooling over an aligned window. More...
float	compute_region_coordinate (int p, float bin_size, float roi_anchor, float max_value)
template<typename input_data_type , typename roi_data_type >
void	roi_align (const ITensor *input, ITensor *output, const ITensor *rois, ROIPoolingLayerInfo pool_info, const Window &window, const ThreadInfo &info)
template void	roi_align< float, float > (const ITensor *input, ITensor *output, const ITensor *rois, ROIPoolingLayerInfo pool_info, const Window &window, const ThreadInfo &info)
template void	roi_align< uint8_t, uint16_t > (const ITensor *input, ITensor *output, const ITensor *rois, ROIPoolingLayerInfo pool_info, const Window &window, const ThreadInfo &info)
template void	roi_align< int8_t, uint16_t > (const ITensor *input, ITensor *output, const ITensor *rois, ROIPoolingLayerInfo pool_info, const Window &window, const ThreadInfo &info)
void	neon_qu8_roialign (const ITensor *input, ITensor *output, const ITensor *rois, ROIPoolingLayerInfo pool_info, const Window &window, const ThreadInfo &info)
void	neon_qs8_roialign (const ITensor *input, ITensor *output, const ITensor *rois, ROIPoolingLayerInfo pool_info, const Window &window, const ThreadInfo &info)
void	neon_fp16_roialign (const ITensor *input, ITensor *output, const ITensor *rois, ROIPoolingLayerInfo pool_info, const Window &window, const ThreadInfo &info)
void	u8_neon_scale (const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, bool align_corners, const Window &window)
void	s16_neon_scale (const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, bool align_corners, const Window &window)
void	qasymm8_neon_scale (const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, bool align_corners, const Window &window)
void	qasymm8_signed_neon_scale (const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, bool align_corners, const Window &window)
template<typename T >
void	nearest_neon_scale (const ITensor *src, ITensor *dst, const ITensor *offsets, float sampling_offset, bool align_corners, const Window &window)
template<typename T >
void	bilinear_neon_scale (const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, bool align_corners, const Window &window)
template<typename T >
void	common_neon_scale (const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, bool align_corners, const Window &window)
void	fp32_sve_scale (const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, bool align_corners, const Window &window)
void	u8_sve_scale (const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, bool align_corners, const Window &window)
void	s16_sve_scale (const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, bool align_corners, const Window &window)
void	fp16_sve_scale (const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, bool align_corners, const Window &window)
void	qasymm8_sve_scale (const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, bool align_corners, const Window &window)
void	qasymm8_signed_sve_scale (const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, bool align_corners, const Window &window)
void	neon_f32_select_same_rank (const ITensor *c, const ITensor *x, const ITensor *y, ITensor *output, const Window &window)
void	neon_f32_select_not_same_rank (const ITensor *c, const ITensor *x, const ITensor *y, ITensor *output, const Window &window)
template<typename ScalarType , typename VectorType >
void	select_op (const ITensor *cond, const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window, const int window_step_x, const int window_start_x, const int window_end_x, const int limit, VectorType(*condition_conversion)(const uint8_t *))
template<typename ScalarType , typename VectorType >
void	select_op_8 (const ITensor *cond, const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template<typename ScalarType , typename VectorType >
void	select_op_16 (const ITensor *cond, const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template<typename ScalarType , typename VectorType >
void	select_op_32 (const ITensor *cond, const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template<typename ScalarType >
void	select_op_not_same_rank (const ITensor *cond, const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	select_op_32< float, uint32x4_t > (const ITensor *cond, const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	select_op_not_same_rank< float > (const ITensor *cond, const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	select_op_8< int8_t, uint8x16_t > (const ITensor *cond, const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	select_op_16< int16_t, uint16x8_t > (const ITensor *cond, const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	select_op_32< int32_t, uint32x4_t > (const ITensor *cond, const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	select_op_not_same_rank< int8_t > (const ITensor *cond, const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	select_op_not_same_rank< int16_t > (const ITensor *cond, const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	select_op_not_same_rank< int32_t > (const ITensor *cond, const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	select_op_8< uint8_t, uint8x16_t > (const ITensor *cond, const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	select_op_16< uint16_t, uint16x8_t > (const ITensor *cond, const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	select_op_32< uint32_t, uint32x4_t > (const ITensor *cond, const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	select_op_not_same_rank< uint8_t > (const ITensor *cond, const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	select_op_not_same_rank< uint16_t > (const ITensor *cond, const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
template void	select_op_not_same_rank< uint32_t > (const ITensor *cond, const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window)
void	neon_s8_select_same_rank (const ITensor *c, const ITensor *x, const ITensor *y, ITensor *output, const Window &window)
void	neon_s16_select_same_rank (const ITensor *c, const ITensor *x, const ITensor *y, ITensor *output, const Window &window)
void	neon_s32_select_same_rank (const ITensor *c, const ITensor *x, const ITensor *y, ITensor *output, const Window &window)
void	neon_s8_select_not_same_rank (const ITensor *c, const ITensor *x, const ITensor *y, ITensor *output, const Window &window)
void	neon_s16_select_not_same_rank (const ITensor *c, const ITensor *x, const ITensor *y, ITensor *output, const Window &window)
void	neon_s32_select_not_same_rank (const ITensor *c, const ITensor *x, const ITensor *y, ITensor *output, const Window &window)
void	neon_u8_select_same_rank (const ITensor *c, const ITensor *x, const ITensor *y, ITensor *output, const Window &window)
void	neon_u16_select_same_rank (const ITensor *c, const ITensor *x, const ITensor *y, ITensor *output, const Window &window)
void	neon_u32_select_same_rank (const ITensor *c, const ITensor *x, const ITensor *y, ITensor *output, const Window &window)
void	neon_u8_select_not_same_rank (const ITensor *c, const ITensor *x, const ITensor *y, ITensor *output, const Window &window)
void	neon_u16_select_not_same_rank (const ITensor *c, const ITensor *x, const ITensor *y, ITensor *output, const Window &window)
void	neon_u32_select_not_same_rank (const ITensor *c, const ITensor *x, const ITensor *y, ITensor *output, const Window &window)
void	neon_f16_select_same_rank (const ITensor *c, const ITensor *x, const ITensor *y, ITensor *output, const Window &window)
void	neon_f16_select_not_same_rank (const ITensor *c, const ITensor *x, const ITensor *y, ITensor *output, const Window &window)
void	neon_fp32_softmax (const ITensor *in, const ITensor *max, void *const tmp, ITensor *out, const float beta, bool is_log, const Window &window)
void	neon_fp32_logits (const ITensor *in, ITensor *out, const Window &window)
template<typename T >
void	neon_logits_1d_max (const ITensor *in, ITensor *out, const Window &window)
template void	neon_logits_1d_max< float > (const ITensor *in, ITensor *out, const Window &window)
template void	neon_logits_1d_max< qasymm8_signed_t > (const ITensor *in, ITensor *out, const Window &window)
template void	neon_logits_1d_max< qasymm8_t > (const ITensor *in, ITensor *out, const Window &window)
template<typename T >
void	neon_softmax_logits_1d_quantized (const ITensor *in, const ITensor *max, void *const tmp, ITensor *out, float beta, bool is_log, const Window &window)
template void	neon_softmax_logits_1d_quantized< qasymm8_signed_t > (const ITensor *in, const ITensor *max, void *const tmp, ITensor *out, float beta, bool is_log, const Window &window)
template void	neon_softmax_logits_1d_quantized< qasymm8_t > (const ITensor *in, const ITensor *max, void *const tmp, ITensor *out, float beta, bool is_log, const Window &window)
template<typename T >
void	neon_softmax_logits_1d_float (const ITensor *in, const ITensor *max, void *const tmp, ITensor *out, const float beta, bool is_log, const Window &window)
template void	neon_softmax_logits_1d_float< float > (const ITensor *in, const ITensor *max, void *const tmp, ITensor *out, const float beta, bool is_log, const Window &window)
void	neon_qasymm8_softmax (const ITensor *in, const ITensor *max, void *const tmp, ITensor *out, const float beta, bool is_log, const Window &window)
void	neon_qasymm8_logits (const ITensor *in, ITensor *out, const Window &window)
void	neon_qasymm8_signed_softmax (const ITensor *in, const ITensor *max, void *const tmp, ITensor *out, const float beta, bool is_log, const Window &window)
void	neon_qasymm8_singed_logits (const ITensor *in, ITensor *out, const Window &window)
void	sve_fp32_softmax (const ITensor *in, const ITensor *max, void *const tmp, ITensor *out, const float beta, bool is_log, const Window &window)
void	sve_fp32_logits (const ITensor *in, ITensor *out, const Window &window)
template<typename ScalarType >
void	sve_logits_1d_max (const ITensor *in, ITensor *out, const Window &window)
template<typename ScalarType >
void	sve_softmax_logits_1d_float (const ITensor *in, const ITensor *max, void *const tmp, ITensor *out, const float beta, bool is_log, const Window &window)
template void	sve_logits_1d_max< float > (const ITensor *in, ITensor *out, const Window &window)
template void	sve_logits_1d_max< float16_t > (const ITensor *in, ITensor *out, const Window &window)
template void	sve_logits_1d_max< qasymm8_t > (const ITensor *in, ITensor *out, const Window &window)
template void	sve_logits_1d_max< qasymm8_signed_t > (const ITensor *in, ITensor *out, const Window &window)
template void	sve_softmax_logits_1d_float< float > (const ITensor *in, const ITensor *max, void *const tmp, ITensor *out, const float beta, bool is_log, const Window &window)
template void	sve_softmax_logits_1d_float< float16_t > (const ITensor *in, const ITensor *max, void *const tmp, ITensor *out, const float beta, bool is_log, const Window &window)
void	sve_qasymm8_logits (const ITensor *in, ITensor *out, const Window &window)
void	sve_qasymm8_signed_logits (const ITensor *in, ITensor *out, const Window &window)
template<typename ScalarType >
void	sve2_softmax_logits_1d_quantized (const ITensor *in, const ITensor *max, void *const tmp, ITensor *out, float beta, bool is_log, const Window &window)
template void	sve2_softmax_logits_1d_quantized< qasymm8_signed_t > (const ITensor *in, const ITensor *max, void *const tmp, ITensor *out, float beta, bool is_log, const Window &window)
template void	sve2_softmax_logits_1d_quantized< qasymm8_t > (const ITensor *in, const ITensor *max, void *const tmp, ITensor *out, float beta, bool is_log, const Window &window)
void	sve2_qasymm8_softmax (const ITensor *in, const ITensor *max, void *const tmp, ITensor *out, const float beta, bool is_log, const Window &window)
void	sve2_qasymm8_signed_softmax (const ITensor *in, const ITensor *max, void *const tmp, ITensor *out, const float beta, bool is_log, const Window &window)
void	neon_fp16_softmax (const ITensor *in, const ITensor *max, void *const tmp, ITensor *out, const float beta, bool is_log, const Window &window)
void	sve_fp16_softmax (const ITensor *in, const ITensor *max, void *const tmp, ITensor *out, const float beta, bool is_log, const Window &window)
void	neon_fp16_logits (const ITensor *in, ITensor *out, const Window &window)
void	sve_fp16_logits (const ITensor *in, ITensor *out, const Window &window)
void	sub_qasymm8_neon (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
void	sub_qasymm8_signed_neon (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
void	sub_qsymm16_neon (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)
template<typename T >
void	sub_same_neon (const ITensor *src0, const ITensor *src1, ITensor *dst, const ConvertPolicy &policy, const Window &window)

Variables
constexpr int	step = 4

Typedef Documentation

CpuElementwiseMax

using CpuElementwiseMax = CpuElementwiseArithmetic<ArithmeticOperation::MAX>

Class to run cpu::kernels::CpuArithmeticKernel except for maximum operation.

Definition at line 65 of file CpuElementwise.h.

CpuElementwiseMin

using CpuElementwiseMin = CpuElementwiseArithmetic<ArithmeticOperation::MIN>

Class to run cpu::kernels::CpuArithmeticKernel except for minimum operation.

Definition at line 67 of file CpuElementwise.h.

CpuElementwiseSquaredDiff

using CpuElementwiseSquaredDiff = CpuElementwiseArithmetic<ArithmeticOperation::SQUARED_DIFF>

Class to run cpu::kernels::CpuArithmeticKernel except for squared difference operation.

Definition at line 69 of file CpuElementwise.h.

CpuLogSoftmax

using CpuLogSoftmax = CpuSoftmaxGeneric<true>

Definition at line 107 of file CpuSoftmax.h.

CpuPRelu

using CpuPRelu = CpuElementwiseArithmetic<ArithmeticOperation::PRELU>

Class to run cpu::kernels::CpuArithmeticKernel except for PRelu operation.

Definition at line 34 of file CpuPRelu.h.

CpuSoftmax

using CpuSoftmax = CpuSoftmaxGeneric<false>

Definition at line 106 of file CpuSoftmax.h.

ICpuOperator

using ICpuOperator = experimental::INEOperator

Definition at line 33 of file ICpuOperator.h.

KernelType

using KernelType = kernels::CpuElementwiseUnaryKernel

Definition at line 33 of file CpuElementwiseUnary.cpp.

NEEqual

using NEEqual = CpuElementwiseComparisonStatic<ComparisonOperation::Equal>

Basic function to run equal comparison.

Definition at line 171 of file CpuElementwise.h.

NEGreater

using NEGreater = CpuElementwiseComparisonStatic<ComparisonOperation::Greater>

Basic function to run greater comparison.

Definition at line 175 of file CpuElementwise.h.

NEGreaterEqual

using NEGreaterEqual = CpuElementwiseComparisonStatic<ComparisonOperation::GreaterEqual>

Basic function to run greater-equal comparison.

Definition at line 177 of file CpuElementwise.h.

NELess

using NELess = CpuElementwiseComparisonStatic<ComparisonOperation::Less>

Basic function to run less comparison.

Definition at line 179 of file CpuElementwise.h.

NELessEqual

using NELessEqual = CpuElementwiseComparisonStatic<ComparisonOperation::LessEqual>

Basic function to run less-equal comparison.

Definition at line 181 of file CpuElementwise.h.

NENotEqual

using NENotEqual = CpuElementwiseComparisonStatic<ComparisonOperation::NotEqual>

Basic function to run not equal comparison.

Definition at line 173 of file CpuElementwise.h.

Enumeration Type Documentation

AsmConvMethod

enum AsmConvMethod

strong

Enumerator
Im2Col
Indirect
Conv

Definition at line 35 of file CpuGemmAssemblyDispatch.h.

{
    Im2Col,
    Indirect,
    Conv
};

KernelSelectionType

enum KernelSelectionType

strong

Enumerator
Preferred	Retrieve the best implementation available for the given Cpu ISA, ignoring the build flags.
Supported	Retrieve the best implementation available for the given Cpu ISA that is supported by the current build.

Definition at line 34 of file ICpuKernel.h.

{
    Preferred, /**< Retrieve the best implementation available for the given Cpu ISA, ignoring the build flags */
    Supported  /**< Retrieve the best implementation available for the given Cpu ISA that is supported by the current build */
};

Function Documentation

add_fp16_neon()

void arm_compute::cpu::add_fp16_neon	(	const ITensor *	src0,
		const ITensor *	src1,
		ITensor *	dst,
		const ConvertPolicy &	policy,
		const Window &	window
	)

Referenced by arm_compute::cpu::kernels::can_interpret_inputs_as_1d_array().

add_fp16_neon_as_1d_array()

void arm_compute::cpu::add_fp16_neon_as_1d_array	(	const ITensor *	src0,
		const ITensor *	src1,
		ITensor *	dst,
		const ConvertPolicy &	policy,
		const Window &	window
	)

Referenced by arm_compute::cpu::kernels::can_interpret_inputs_as_1d_array().

add_fp16_sve()

void arm_compute::cpu::add_fp16_sve	(	const ITensor *	src0,
		const ITensor *	src1,
		ITensor *	dst,
		const ConvertPolicy &	policy,
		const Window &	window
	)

Referenced by arm_compute::cpu::kernels::can_interpret_inputs_as_1d_array().

add_fp32_neon()

void add_fp32_neon	(	const ITensor *	src0,
		const ITensor *	src1,
		ITensor *	dst,
		const ConvertPolicy &	policy,
		const Window &	window
	)

Definition at line 31 of file fp32.cpp.

References add_same_neon< float >(), and arm_compute::test::validation::dst.

Referenced by arm_compute::cpu::kernels::can_interpret_inputs_as_1d_array().

{
    return add_same_neon<float>(src0, src1, dst, policy, window);
}

add_fp32_neon_as_1d_array()

void add_fp32_neon_as_1d_array	(	const ITensor *	src0,
		const ITensor *	src1,
		ITensor *	dst,
		const ConvertPolicy &	policy,
		const Window &	window
	)

Definition at line 36 of file fp32.cpp.

References add_same_neon_as_1d_array< float >(), and arm_compute::test::validation::dst.

Referenced by arm_compute::cpu::kernels::can_interpret_inputs_as_1d_array().

{
    return add_same_neon_as_1d_array<float>(src0, src1, dst, policy, window);
}

add_fp32_sve()

void add_fp32_sve	(	const ITensor *	src0,
		const ITensor *	src1,
		ITensor *	dst,
		const ConvertPolicy &	policy,
		const Window &	window
	)

Definition at line 33 of file fp32.cpp.

References add_same_sve< float >(), and arm_compute::test::validation::dst.

Referenced by arm_compute::cpu::kernels::can_interpret_inputs_as_1d_array().

{
    return add_same_sve<float>(src0, src1, dst, policy, window);
}

add_qasymm8_neon()

void add_qasymm8_neon	(	const ITensor *	src0,
		const ITensor *	src1,
		ITensor *	dst,
		const ConvertPolicy &	policy,
		const Window &	window
	)

Definition at line 35 of file qasymm8.cpp.

References ARM_COMPUTE_UNUSED, arm_compute::test::validation::b, arm_compute::graph::bfs(), Window::broadcast_if_dimension_le_one(), Window::DimX, Window::Dimension::end(), arm_compute::execute_window_loop(), ITensor::info(), UniformQuantizationInfo::offset, Iterator::ptr(), ITensorInfo::quantization_info(), arm_compute::quantize_qasymm8(), UniformQuantizationInfo::scale, Window::set(), Window::Dimension::start(), Window::Dimension::step(), ITensorInfo::tensor_shape(), QuantizationInfo::uniform(), Dimensions< T >::x(), and Window::x().

Referenced by arm_compute::cpu::kernels::can_interpret_inputs_as_1d_array().

{
    ARM_COMPUTE_UNUSED(policy);

    // Create input windows
    Window input1_win = window.broadcast_if_dimension_le_one(src0->info()->tensor_shape());
    Window input2_win = window.broadcast_if_dimension_le_one(src1->info()->tensor_shape());

    // Clear X Dimension on execution window as we handle manually
    Window win = window;
    win.set(Window::DimX, Window::Dimension(0, 1, 1));

    const int  window_step_x         = 16;
    const auto window_start_x        = static_cast<int>(window.x().start());
    const auto window_end_x          = static_cast<int>(window.x().end());
    const bool is_broadcast_across_x = src0->info()->tensor_shape().x() != src1->info()->tensor_shape().x();

    const UniformQuantizationInfo iq1_info = src0->info()->quantization_info().uniform();
    const UniformQuantizationInfo iq2_info = src1->info()->quantization_info().uniform();
    const UniformQuantizationInfo oq_info  = dst->info()->quantization_info().uniform();

    const float32x4_t invvscaleo = vdupq_n_f32(1.f / oq_info.scale);
    const float32x4_t voffseto   = vdupq_n_f32(oq_info.offset);

    if(is_broadcast_across_x)
    {
        const bool                    is_broadcast_input_2 = input2_win.x().step() == 0;
        Window                        broadcast_win        = is_broadcast_input_2 ? input2_win : input1_win;
        Window                        non_broadcast_win    = !is_broadcast_input_2 ? input2_win : input1_win;
        const ITensor                *broadcast_tensor     = is_broadcast_input_2 ? src1 : src0;
        const ITensor                *non_broadcast_tensor = !is_broadcast_input_2 ? src1 : src0;
        const UniformQuantizationInfo broadcast_qinfo      = broadcast_tensor->info()->quantization_info().uniform();
        const UniformQuantizationInfo non_broadcast_qinfo  = non_broadcast_tensor->info()->quantization_info().uniform();

        const float32x4_t vscale1  = is_broadcast_input_2 ? vdupq_n_f32(iq1_info.scale) : vdupq_n_f32(iq2_info.scale);
        const float32x4_t vscale2  = is_broadcast_input_2 ? vdupq_n_f32(iq2_info.scale) : vdupq_n_f32(iq1_info.scale);
        const int32x4_t   voffset1 = is_broadcast_input_2 ? vdupq_n_s32(iq1_info.offset) : vdupq_n_s32(iq2_info.offset);
        const int32x4_t   voffset2 = is_broadcast_input_2 ? vdupq_n_s32(iq2_info.offset) : vdupq_n_s32(iq1_info.offset);

        // Clear X Dimension on execution window as we handle manually
        non_broadcast_win.set(Window::DimX, Window::Dimension(0, 1, 1));

        Iterator broadcast_input(broadcast_tensor, broadcast_win);
        Iterator non_broadcast_input(non_broadcast_tensor, non_broadcast_win);
        Iterator output(dst, win);

        execute_window_loop(win, [&](const Coordinates &)
        {
            const auto non_broadcast_input_ptr = reinterpret_cast<const uint8_t *>(non_broadcast_input.ptr());
            const auto output_ptr              = reinterpret_cast<uint8_t *>(output.ptr());

            const uint8_t    broadcast_value     = *reinterpret_cast<const uint8_t *>(broadcast_input.ptr());
            const uint8x16_t broadcast_value_vec = vdupq_n_u8(broadcast_value);

            const auto bf_0 = vmulq_f32(vcvtq_f32_s32(vsubq_s32(vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(broadcast_value_vec))))), voffset2)), vscale2);
            const auto bf_1 = vmulq_f32(vcvtq_f32_s32(vsubq_s32(vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(vmovl_u8(vget_low_u8(broadcast_value_vec))))), voffset2)), vscale2);
            const auto bf_2 = vmulq_f32(vcvtq_f32_s32(vsubq_s32(vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(vmovl_u8(vget_high_u8(broadcast_value_vec))))), voffset2)), vscale2);
            const auto bf_3 = vmulq_f32(vcvtq_f32_s32(vsubq_s32(vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(vmovl_u8(vget_high_u8(broadcast_value_vec))))), voffset2)), vscale2);

            const float bfs = static_cast<int32_t>(broadcast_value - broadcast_qinfo.offset) * broadcast_qinfo.scale;

            // Compute S elements per iteration
            int x = window_start_x;
            for(; x <= (window_end_x - window_step_x); x += window_step_x)
            {
                const uint8x16_t a    = vld1q_u8(non_broadcast_input_ptr + x);
                const auto       af_0 = vmulq_f32(vcvtq_f32_s32(vsubq_s32(vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(a))))), voffset1)), vscale1);
                const auto       af_1 = vmulq_f32(vcvtq_f32_s32(vsubq_s32(vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(vmovl_u8(vget_low_u8(a))))), voffset1)), vscale1);
                const auto       af_2 = vmulq_f32(vcvtq_f32_s32(vsubq_s32(vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(vmovl_u8(vget_high_u8(a))))), voffset1)), vscale1);
                const auto       af_3 = vmulq_f32(vcvtq_f32_s32(vsubq_s32(vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(vmovl_u8(vget_high_u8(a))))), voffset1)), vscale1);

                int32x4_t rf_0{};
                int32x4_t rf_1{};
                int32x4_t rf_2{};
                int32x4_t rf_3{};

#ifdef __aarch64__
                rf_0 = vcvtnq_s32_f32(vmlaq_f32(voffseto, vaddq_f32(af_0, bf_0), invvscaleo));
                rf_1 = vcvtnq_s32_f32(vmlaq_f32(voffseto, vaddq_f32(af_1, bf_1), invvscaleo));
                rf_2 = vcvtnq_s32_f32(vmlaq_f32(voffseto, vaddq_f32(af_2, bf_2), invvscaleo));
                rf_3 = vcvtnq_s32_f32(vmlaq_f32(voffseto, vaddq_f32(af_3, bf_3), invvscaleo));
#else  //__aarch64__
                rf_0 = vcvtq_s32_f32(vmlaq_f32(voffseto, vaddq_f32(af_0, bf_0), invvscaleo));
                rf_1 = vcvtq_s32_f32(vmlaq_f32(voffseto, vaddq_f32(af_1, bf_1), invvscaleo));
                rf_2 = vcvtq_s32_f32(vmlaq_f32(voffseto, vaddq_f32(af_2, bf_2), invvscaleo));
                rf_3 = vcvtq_s32_f32(vmlaq_f32(voffseto, vaddq_f32(af_3, bf_3), invvscaleo));
#endif //__aarch64__

                const uint8x8_t pa = vqmovun_s16(vcombine_s16(vqmovn_s32(rf_0), vqmovn_s32(rf_1)));
                const uint8x8_t pb = vqmovun_s16(vcombine_s16(vqmovn_s32(rf_2), vqmovn_s32(rf_3)));
                vst1q_u8(output_ptr + x, vcombine_u8(pa, pb));
            }

            // Compute left-over elements
            for(; x < window_end_x; ++x)
            {
                const float afs   = static_cast<int32_t>(*(non_broadcast_input_ptr + x) - non_broadcast_qinfo.offset) * non_broadcast_qinfo.scale;
                *(output_ptr + x) = quantize_qasymm8((afs + bfs), oq_info);
            }
        },
        broadcast_input, non_broadcast_input, output);
    }
    else
    {
        // Clear X Dimension on execution window as we handle manually
        input1_win.set(Window::DimX, Window::Dimension(0, 1, 1));
        input2_win.set(Window::DimX, Window::Dimension(0, 1, 1));

        Iterator input1(src0, input1_win);
        Iterator input2(src1, input2_win);
        Iterator output(dst, win);

        const float32x4_t vscale1  = vdupq_n_f32(iq1_info.scale);
        const float32x4_t vscale2  = vdupq_n_f32(iq2_info.scale);
        const int32x4_t   voffset1 = vdupq_n_s32(iq1_info.offset);
        const int32x4_t   voffset2 = vdupq_n_s32(iq2_info.offset);

        execute_window_loop(win, [&](const Coordinates &)
        {
            const auto input1_ptr = reinterpret_cast<const uint8_t *>(input1.ptr());
            const auto input2_ptr = reinterpret_cast<const uint8_t *>(input2.ptr());
            const auto output_ptr = reinterpret_cast<uint8_t *>(output.ptr());

            // Compute S elements per iteration
            int x = window_start_x;
            for(; x <= (window_end_x - window_step_x); x += window_step_x)
            {
                const uint8x16_t a = vld1q_u8(input1_ptr + x);
                const uint8x16_t b = vld1q_u8(input2_ptr + x);

                const auto af_0 = vmulq_f32(vcvtq_f32_s32(vsubq_s32(vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(a))))), voffset1)), vscale1);
                const auto af_1 = vmulq_f32(vcvtq_f32_s32(vsubq_s32(vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(vmovl_u8(vget_low_u8(a))))), voffset1)), vscale1);
                const auto af_2 = vmulq_f32(vcvtq_f32_s32(vsubq_s32(vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(vmovl_u8(vget_high_u8(a))))), voffset1)), vscale1);
                const auto af_3 = vmulq_f32(vcvtq_f32_s32(vsubq_s32(vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(vmovl_u8(vget_high_u8(a))))), voffset1)), vscale1);

                const auto bf_0 = vmulq_f32(vcvtq_f32_s32(vsubq_s32(vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(b))))), voffset2)), vscale2);
                const auto bf_1 = vmulq_f32(vcvtq_f32_s32(vsubq_s32(vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(vmovl_u8(vget_low_u8(b))))), voffset2)), vscale2);
                const auto bf_2 = vmulq_f32(vcvtq_f32_s32(vsubq_s32(vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(vmovl_u8(vget_high_u8(b))))), voffset2)), vscale2);
                const auto bf_3 = vmulq_f32(vcvtq_f32_s32(vsubq_s32(vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(vmovl_u8(vget_high_u8(b))))), voffset2)), vscale2);

                int32x4_t rf_0{};
                int32x4_t rf_1{};
                int32x4_t rf_2{};
                int32x4_t rf_3{};

#ifdef __aarch64__
                rf_0 = vcvtnq_s32_f32(vmlaq_f32(voffseto, vaddq_f32(af_0, bf_0), invvscaleo));
                rf_1 = vcvtnq_s32_f32(vmlaq_f32(voffseto, vaddq_f32(af_1, bf_1), invvscaleo));
                rf_2 = vcvtnq_s32_f32(vmlaq_f32(voffseto, vaddq_f32(af_2, bf_2), invvscaleo));
                rf_3 = vcvtnq_s32_f32(vmlaq_f32(voffseto, vaddq_f32(af_3, bf_3), invvscaleo));
#else  //__aarch64__
                rf_0 = vcvtq_s32_f32(vmlaq_f32(voffseto, vaddq_f32(af_0, bf_0), invvscaleo));
                rf_1 = vcvtq_s32_f32(vmlaq_f32(voffseto, vaddq_f32(af_1, bf_1), invvscaleo));
                rf_2 = vcvtq_s32_f32(vmlaq_f32(voffseto, vaddq_f32(af_2, bf_2), invvscaleo));
                rf_3 = vcvtq_s32_f32(vmlaq_f32(voffseto, vaddq_f32(af_3, bf_3), invvscaleo));
#endif //__aarch64__

                const uint8x8_t pa = vqmovun_s16(vcombine_s16(vqmovn_s32(rf_0), vqmovn_s32(rf_1)));
                const uint8x8_t pb = vqmovun_s16(vcombine_s16(vqmovn_s32(rf_2), vqmovn_s32(rf_3)));
                vst1q_u8(output_ptr + x, vcombine_u8(pa, pb));
            }

            // Compute left-over elements
            for(; x < window_end_x; ++x)
            {
                const float afs   = static_cast<int32_t>((*(input1_ptr + x)) - iq1_info.offset) * iq1_info.scale;
                const float bfs   = static_cast<int32_t>((*(input2_ptr + x)) - iq2_info.offset) * iq2_info.scale;
                *(output_ptr + x) = quantize_qasymm8((afs + bfs), oq_info);
            }
        },
        input1, input2, output);
    }
}

add_qasymm8_signed_neon()

void add_qasymm8_signed_neon	(	const ITensor *	src0,
		const ITensor *	src1,
		ITensor *	dst,
		const ConvertPolicy &	policy,
		const Window &	window
	)

Definition at line 35 of file qasymm8_signed.cpp.

References ARM_COMPUTE_UNUSED, arm_compute::test::validation::b, arm_compute::graph::bfs(), Window::broadcast_if_dimension_le_one(), Window::DimX, Window::Dimension::end(), arm_compute::execute_window_loop(), ITensor::info(), UniformQuantizationInfo::offset, Iterator::ptr(), ITensorInfo::quantization_info(), arm_compute::quantize_qasymm8_signed(), UniformQuantizationInfo::scale, Window::set(), Window::Dimension::start(), Window::Dimension::step(), ITensorInfo::tensor_shape(), QuantizationInfo::uniform(), Dimensions< T >::x(), and Window::x().

Referenced by arm_compute::cpu::kernels::can_interpret_inputs_as_1d_array().

{
    ARM_COMPUTE_UNUSED(policy);

    // Create input windows
    Window input1_win = window.broadcast_if_dimension_le_one(src0->info()->tensor_shape());
    Window input2_win = window.broadcast_if_dimension_le_one(src1->info()->tensor_shape());

    // Clear X Dimension on execution window as we handle manually
    Window win = window;
    win.set(Window::DimX, Window::Dimension(0, 1, 1));

    const int  window_step_x         = 16;
    const auto window_start_x        = static_cast<int>(window.x().start());
    const auto window_end_x          = static_cast<int>(window.x().end());
    const bool is_broadcast_across_x = src0->info()->tensor_shape().x() != src1->info()->tensor_shape().x();

    const UniformQuantizationInfo iq1_info = src0->info()->quantization_info().uniform();
    const UniformQuantizationInfo iq2_info = src1->info()->quantization_info().uniform();
    const UniformQuantizationInfo oq_info  = dst->info()->quantization_info().uniform();

    const float32x4_t invvscaleo = vdupq_n_f32(1.f / oq_info.scale);
    const float32x4_t voffseto   = vdupq_n_f32(oq_info.offset);

    if(is_broadcast_across_x)
    {
        const bool                    is_broadcast_input_2 = input2_win.x().step() == 0;
        Window                        broadcast_win        = is_broadcast_input_2 ? input2_win : input1_win;
        Window                        non_broadcast_win    = !is_broadcast_input_2 ? input2_win : input1_win;
        const ITensor                *broadcast_tensor     = is_broadcast_input_2 ? src1 : src0;
        const ITensor                *non_broadcast_tensor = !is_broadcast_input_2 ? src1 : src0;
        const UniformQuantizationInfo broadcast_qinfo      = broadcast_tensor->info()->quantization_info().uniform();
        const UniformQuantizationInfo non_broadcast_qinfo  = non_broadcast_tensor->info()->quantization_info().uniform();

        const float32x4_t vscale1  = is_broadcast_input_2 ? vdupq_n_f32(iq1_info.scale) : vdupq_n_f32(iq2_info.scale);
        const float32x4_t vscale2  = is_broadcast_input_2 ? vdupq_n_f32(iq2_info.scale) : vdupq_n_f32(iq1_info.scale);
        const int32x4_t   voffset1 = is_broadcast_input_2 ? vdupq_n_s32(iq1_info.offset) : vdupq_n_s32(iq2_info.offset);
        const int32x4_t   voffset2 = is_broadcast_input_2 ? vdupq_n_s32(iq2_info.offset) : vdupq_n_s32(iq1_info.offset);

        // Clear X Dimension on execution window as we handle manually
        non_broadcast_win.set(Window::DimX, Window::Dimension(0, 1, 1));

        Iterator broadcast_input(broadcast_tensor, broadcast_win);
        Iterator non_broadcast_input(non_broadcast_tensor, non_broadcast_win);
        Iterator output(dst, win);

        execute_window_loop(win, [&](const Coordinates &)
        {
            const auto non_broadcast_input_ptr = reinterpret_cast<const int8_t *>(non_broadcast_input.ptr());
            const auto output_ptr              = reinterpret_cast<int8_t *>(output.ptr());

            const int8_t    broadcast_value     = *reinterpret_cast<const int8_t *>(broadcast_input.ptr());
            const int8x16_t broadcast_value_vec = vdupq_n_s8(broadcast_value);

            const auto  bf_0 = vmulq_f32(vcvtq_f32_s32(vsubq_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_low_s8(broadcast_value_vec)))), voffset2)), vscale2);
            const auto  bf_1 = vmulq_f32(vcvtq_f32_s32(vsubq_s32(vmovl_s16(vget_high_s16(vmovl_s8(vget_low_s8(broadcast_value_vec)))), voffset2)), vscale2);
            const auto  bf_2 = vmulq_f32(vcvtq_f32_s32(vsubq_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_high_s8(broadcast_value_vec)))), voffset2)), vscale2);
            const auto  bf_3 = vmulq_f32(vcvtq_f32_s32(vsubq_s32(vmovl_s16(vget_high_s16(vmovl_s8(vget_high_s8(broadcast_value_vec)))), voffset2)), vscale2);
            const float bfs  = static_cast<int32_t>(broadcast_value - broadcast_qinfo.offset) * broadcast_qinfo.scale;

            // Compute S elements per iteration
            int x = window_start_x;
            for(; x <= (window_end_x - window_step_x); x += window_step_x)
            {
                const int8x16_t a = vld1q_s8(non_broadcast_input_ptr + x);

                const auto af_0 = vmulq_f32(vcvtq_f32_s32(vsubq_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_low_s8(a)))), voffset1)), vscale1);
                const auto af_1 = vmulq_f32(vcvtq_f32_s32(vsubq_s32(vmovl_s16(vget_high_s16(vmovl_s8(vget_low_s8(a)))), voffset1)), vscale1);
                const auto af_2 = vmulq_f32(vcvtq_f32_s32(vsubq_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_high_s8(a)))), voffset1)), vscale1);
                const auto af_3 = vmulq_f32(vcvtq_f32_s32(vsubq_s32(vmovl_s16(vget_high_s16(vmovl_s8(vget_high_s8(a)))), voffset1)), vscale1);

                int32x4_t rf_0{};
                int32x4_t rf_1{};
                int32x4_t rf_2{};
                int32x4_t rf_3{};

#ifdef __aarch64__
                rf_0 = vcvtnq_s32_f32(vmlaq_f32(voffseto, vaddq_f32(af_0, bf_0), invvscaleo));
                rf_1 = vcvtnq_s32_f32(vmlaq_f32(voffseto, vaddq_f32(af_1, bf_1), invvscaleo));
                rf_2 = vcvtnq_s32_f32(vmlaq_f32(voffseto, vaddq_f32(af_2, bf_2), invvscaleo));
                rf_3 = vcvtnq_s32_f32(vmlaq_f32(voffseto, vaddq_f32(af_3, bf_3), invvscaleo));
#else  //__aarch64__
                rf_0 = vcvtq_s32_f32(vmlaq_f32(voffseto, vaddq_f32(af_0, bf_0), invvscaleo));
                rf_1 = vcvtq_s32_f32(vmlaq_f32(voffseto, vaddq_f32(af_1, bf_1), invvscaleo));
                rf_2 = vcvtq_s32_f32(vmlaq_f32(voffseto, vaddq_f32(af_2, bf_2), invvscaleo));
                rf_3 = vcvtq_s32_f32(vmlaq_f32(voffseto, vaddq_f32(af_3, bf_3), invvscaleo));
#endif //__aarch64__

                const int8x8_t pa = vqmovn_s16(vcombine_s16(vqmovn_s32(rf_0), vqmovn_s32(rf_1)));
                const int8x8_t pb = vqmovn_s16(vcombine_s16(vqmovn_s32(rf_2), vqmovn_s32(rf_3)));
                vst1q_s8(output_ptr + x, vcombine_s8(pa, pb));
            }

            // Compute left-over elements
            for(; x < window_end_x; ++x)
            {
                const float afs   = static_cast<int32_t>(*(non_broadcast_input_ptr + x) - non_broadcast_qinfo.offset) * non_broadcast_qinfo.scale;
                *(output_ptr + x) = quantize_qasymm8_signed((afs + bfs), oq_info);
            }
        },
        broadcast_input, non_broadcast_input, output);
    }
    else
    {
        // Clear X Dimension on execution window as we handle manually
        input1_win.set(Window::DimX, Window::Dimension(0, 1, 1));
        input2_win.set(Window::DimX, Window::Dimension(0, 1, 1));

        Iterator input1(src0, input1_win);
        Iterator input2(src1, input2_win);
        Iterator output(dst, win);

        const float32x4_t vscale1  = vdupq_n_f32(iq1_info.scale);
        const float32x4_t vscale2  = vdupq_n_f32(iq2_info.scale);
        const int32x4_t   voffset1 = vdupq_n_s32(iq1_info.offset);
        const int32x4_t   voffset2 = vdupq_n_s32(iq2_info.offset);
        execute_window_loop(win, [&](const Coordinates &)
        {
            const auto input1_ptr = reinterpret_cast<const int8_t *>(input1.ptr());
            const auto input2_ptr = reinterpret_cast<const int8_t *>(input2.ptr());
            const auto output_ptr = reinterpret_cast<int8_t *>(output.ptr());

            // Compute S elements per iteration
            int x = window_start_x;
            for(; x <= (window_end_x - window_step_x); x += window_step_x)
            {
                const int8x16_t a = vld1q_s8(input1_ptr + x);
                const int8x16_t b = vld1q_s8(input2_ptr + x);

                const auto af_0 = vmulq_f32(vcvtq_f32_s32(vsubq_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_low_s8(a)))), voffset1)), vscale1);
                const auto af_1 = vmulq_f32(vcvtq_f32_s32(vsubq_s32(vmovl_s16(vget_high_s16(vmovl_s8(vget_low_s8(a)))), voffset1)), vscale1);
                const auto af_2 = vmulq_f32(vcvtq_f32_s32(vsubq_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_high_s8(a)))), voffset1)), vscale1);
                const auto af_3 = vmulq_f32(vcvtq_f32_s32(vsubq_s32(vmovl_s16(vget_high_s16(vmovl_s8(vget_high_s8(a)))), voffset1)), vscale1);

                const auto bf_0 = vmulq_f32(vcvtq_f32_s32(vsubq_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_low_s8(b)))), voffset2)), vscale2);
                const auto bf_1 = vmulq_f32(vcvtq_f32_s32(vsubq_s32(vmovl_s16(vget_high_s16(vmovl_s8(vget_low_s8(b)))), voffset2)), vscale2);
                const auto bf_2 = vmulq_f32(vcvtq_f32_s32(vsubq_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_high_s8(b)))), voffset2)), vscale2);
                const auto bf_3 = vmulq_f32(vcvtq_f32_s32(vsubq_s32(vmovl_s16(vget_high_s16(vmovl_s8(vget_high_s8(b)))), voffset2)), vscale2);

                int32x4_t rf_0{};
                int32x4_t rf_1{};
                int32x4_t rf_2{};
                int32x4_t rf_3{};

#ifdef __aarch64__
                rf_0 = vcvtnq_s32_f32(vmlaq_f32(voffseto, vaddq_f32(af_0, bf_0), invvscaleo));
                rf_1 = vcvtnq_s32_f32(vmlaq_f32(voffseto, vaddq_f32(af_1, bf_1), invvscaleo));
                rf_2 = vcvtnq_s32_f32(vmlaq_f32(voffseto, vaddq_f32(af_2, bf_2), invvscaleo));
                rf_3 = vcvtnq_s32_f32(vmlaq_f32(voffseto, vaddq_f32(af_3, bf_3), invvscaleo));
#else  //__aarch64__
                rf_0 = vcvtq_s32_f32(vmlaq_f32(voffseto, vaddq_f32(af_0, bf_0), invvscaleo));
                rf_1 = vcvtq_s32_f32(vmlaq_f32(voffseto, vaddq_f32(af_1, bf_1), invvscaleo));
                rf_2 = vcvtq_s32_f32(vmlaq_f32(voffseto, vaddq_f32(af_2, bf_2), invvscaleo));
                rf_3 = vcvtq_s32_f32(vmlaq_f32(voffseto, vaddq_f32(af_3, bf_3), invvscaleo));
#endif //__aarch64__

                const int8x8_t pa = vqmovn_s16(vcombine_s16(vqmovn_s32(rf_0), vqmovn_s32(rf_1)));
                const int8x8_t pb = vqmovn_s16(vcombine_s16(vqmovn_s32(rf_2), vqmovn_s32(rf_3)));
                vst1q_s8(output_ptr + x, vcombine_s8(pa, pb));
            }

            // Compute left-over elements
            for(; x < window_end_x; ++x)
            {
                const float afs   = static_cast<int32_t>((*(input1_ptr + x)) - iq1_info.offset) * iq1_info.scale;
                const float bfs   = static_cast<int32_t>((*(input2_ptr + x)) - iq2_info.offset) * iq2_info.scale;
                *(output_ptr + x) = quantize_qasymm8_signed((afs + bfs), dst->info()->quantization_info());
            }
        },
        input1, input2, output);
    }
}

add_qasymm8_signed_sve2()

void add_qasymm8_signed_sve2	(	const ITensor *	src0,
		const ITensor *	src1,
		ITensor *	dst,
		const ConvertPolicy &	policy,
		const Window &	window
	)

Definition at line 37 of file qasymm8_signed.cpp.

References ARM_COMPUTE_UNUSED, arm_compute::test::validation::b, Window::broadcast_if_dimension_le_one(), Window::DimX, Window::Dimension::end(), arm_compute::execute_window_loop(), ITensor::info(), UniformQuantizationInfo::offset, Iterator::ptr(), ITensorInfo::quantization_info(), UniformQuantizationInfo::scale, Window::set(), Window::Dimension::start(), Window::Dimension::step(), ITensorInfo::tensor_shape(), QuantizationInfo::uniform(), Dimensions< T >::x(), and Window::x().

Referenced by arm_compute::cpu::kernels::can_interpret_inputs_as_1d_array().

{
    ARM_COMPUTE_UNUSED(policy);

    // Create input windows
    Window input1_win = window.broadcast_if_dimension_le_one(src0->info()->tensor_shape());
    Window input2_win = window.broadcast_if_dimension_le_one(src1->info()->tensor_shape());

    // Clear X Dimension on execution window as we handle manually
    Window win = window;
    win.set(Window::DimX, Window::Dimension(0, 1, 1));

    const auto window_start_x        = static_cast<int>(window.x().start());
    const auto window_end_x          = static_cast<int>(window.x().end());
    const bool is_broadcast_across_x = src0->info()->tensor_shape().x() != src1->info()->tensor_shape().x();

    const UniformQuantizationInfo iq1_info = src0->info()->quantization_info().uniform();
    const UniformQuantizationInfo iq2_info = src1->info()->quantization_info().uniform();
    const UniformQuantizationInfo oq_info  = dst->info()->quantization_info().uniform();

    const auto invvscaleo = svdup_n_f32(1.f / oq_info.scale);
    const auto voffseto   = svdup_n_f32(oq_info.offset);

    if(is_broadcast_across_x)
    {
        const bool     is_broadcast_input_2 = input2_win.x().step() == 0;
        Window         broadcast_win        = is_broadcast_input_2 ? input2_win : input1_win;
        Window         non_broadcast_win    = !is_broadcast_input_2 ? input2_win : input1_win;
        const ITensor *broadcast_tensor     = is_broadcast_input_2 ? src1 : src0;
        const ITensor *non_broadcast_tensor = !is_broadcast_input_2 ? src1 : src0;
        const auto     all_true_pg          = svptrue_b8();

        const auto vscale1  = is_broadcast_input_2 ? svdup_n_f32(iq1_info.scale) : svdup_n_f32(iq2_info.scale);
        const auto vscale2  = is_broadcast_input_2 ? svdup_n_f32(iq2_info.scale) : svdup_n_f32(iq1_info.scale);
        const auto voffset1 = is_broadcast_input_2 ? svdup_n_s32(iq1_info.offset) : svdup_n_s32(iq2_info.offset);
        const auto voffset2 = is_broadcast_input_2 ? svdup_n_s32(iq2_info.offset) : svdup_n_s32(iq1_info.offset);

        // Clear X Dimension on execution window as we handle manually
        non_broadcast_win.set(Window::DimX, Window::Dimension(0, 1, 1));

        Iterator broadcast_input(broadcast_tensor, broadcast_win);
        Iterator non_broadcast_input(non_broadcast_tensor, non_broadcast_win);
        Iterator output(dst, win);

        execute_window_loop(win, [&](const Coordinates &)
        {
            const auto non_broadcast_input_ptr = reinterpret_cast<const int8_t *>(non_broadcast_input.ptr());
            const auto output_ptr              = reinterpret_cast<int8_t *>(output.ptr());

            const int8_t broadcast_value     = *reinterpret_cast<const int8_t *>(broadcast_input.ptr());
            const auto   broadcast_value_vec = svdup_n_s8(broadcast_value);

            int        x    = window_start_x;
            svbool_t   pg   = svwhilelt_b8(x, window_end_x);
            const auto bf_0 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svmovlb_s32(svmovlb_s16(broadcast_value_vec)), voffset2)), vscale2);
            const auto bf_1 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svmovlt_s32(svmovlb_s16(broadcast_value_vec)), voffset2)), vscale2);
            const auto bf_2 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svmovlb_s32(svmovlt_s16(broadcast_value_vec)), voffset2)), vscale2);
            const auto bf_3 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svmovlt_s32(svmovlt_s16(broadcast_value_vec)), voffset2)), vscale2);

            do
            {
                const auto a    = svld1_s8(pg, non_broadcast_input_ptr + x);
                const auto af_0 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svmovlb_s32(svmovlb_s16(a)), voffset1)), vscale1);
                const auto af_1 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svmovlt_s32(svmovlb_s16(a)), voffset1)), vscale1);
                const auto af_2 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svmovlb_s32(svmovlt_s16(a)), voffset1)), vscale1);
                const auto af_3 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svmovlt_s32(svmovlt_s16(a)), voffset1)), vscale1);

                const auto rf_0 = svcvt_s32_f32_z(pg, svmla_f32_z(pg, voffseto, svadd_f32_z(pg, af_0, bf_0), invvscaleo));
                const auto rf_1 = svcvt_s32_f32_z(pg, svmla_f32_z(pg, voffseto, svadd_f32_z(pg, af_1, bf_1), invvscaleo));
                const auto rf_2 = svcvt_s32_f32_z(pg, svmla_f32_z(pg, voffseto, svadd_f32_z(pg, af_2, bf_2), invvscaleo));
                const auto rf_3 = svcvt_s32_f32_z(pg, svmla_f32_z(pg, voffseto, svadd_f32_z(pg, af_3, bf_3), invvscaleo));

                const auto pa  = svqxtnt_s32(svqxtnb_s32(rf_0), rf_1);
                const auto pb  = svqxtnt_s32(svqxtnb_s32(rf_2), rf_3);
                const auto res = svqxtnt_s16(svqxtnb_s16(pa), pb);

                svst1_s8(pg, output_ptr + x, res);

                x += svcntb();
                pg = svwhilelt_b8(x, window_end_x);
            }
            while(svptest_any(all_true_pg, pg));
        },
        broadcast_input, non_broadcast_input, output);
    }
    else
    {
        // Clear X Dimension on execution window as we handle manually
        input1_win.set(Window::DimX, Window::Dimension(0, 1, 1));
        input2_win.set(Window::DimX, Window::Dimension(0, 1, 1));

        Iterator input1(src0, input1_win);
        Iterator input2(src1, input2_win);
        Iterator output(dst, win);

        const auto vscale1  = svdup_n_f32(iq1_info.scale);
        const auto vscale2  = svdup_n_f32(iq2_info.scale);
        const auto voffset1 = svdup_n_s32(iq1_info.offset);
        const auto voffset2 = svdup_n_s32(iq2_info.offset);

        execute_window_loop(win, [&](const Coordinates &)
        {
            const auto input1_ptr = reinterpret_cast<const int8_t *>(input1.ptr());
            const auto input2_ptr = reinterpret_cast<const int8_t *>(input2.ptr());
            const auto output_ptr = reinterpret_cast<int8_t *>(output.ptr());

            int      x  = window_start_x;
            svbool_t pg = svwhilelt_b8(x, window_end_x);
            do
            {
                const auto a = svld1_s8(pg, input1_ptr + x);
                const auto b = svld1_s8(pg, input2_ptr + x);

                const auto af_0 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svmovlb_s32(svmovlb_s16(a)), voffset1)), vscale1);
                const auto af_1 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svmovlt_s32(svmovlb_s16(a)), voffset1)), vscale1);
                const auto af_2 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svmovlb_s32(svmovlt_s16(a)), voffset1)), vscale1);
                const auto af_3 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svmovlt_s32(svmovlt_s16(a)), voffset1)), vscale1);

                const auto bf_0 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svmovlb_s32(svmovlb_s16(b)), voffset2)), vscale2);
                const auto bf_1 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svmovlt_s32(svmovlb_s16(b)), voffset2)), vscale2);
                const auto bf_2 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svmovlb_s32(svmovlt_s16(b)), voffset2)), vscale2);
                const auto bf_3 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svmovlt_s32(svmovlt_s16(b)), voffset2)), vscale2);

                const auto rf_0 = svcvt_s32_f32_z(pg, svmla_f32_z(pg, voffseto, svadd_f32_z(pg, af_0, bf_0), invvscaleo));
                const auto rf_1 = svcvt_s32_f32_z(pg, svmla_f32_z(pg, voffseto, svadd_f32_z(pg, af_1, bf_1), invvscaleo));
                const auto rf_2 = svcvt_s32_f32_z(pg, svmla_f32_z(pg, voffseto, svadd_f32_z(pg, af_2, bf_2), invvscaleo));
                const auto rf_3 = svcvt_s32_f32_z(pg, svmla_f32_z(pg, voffseto, svadd_f32_z(pg, af_3, bf_3), invvscaleo));

                const auto pa  = svqxtnt_s32(svqxtnb_s32(rf_0), rf_1);
                const auto pb  = svqxtnt_s32(svqxtnb_s32(rf_2), rf_3);
                const auto res = svqxtnt_s16(svqxtnb_s16(pa), pb);

                svst1_s8(pg, output_ptr + x, res);

                x += svcntb();
                pg = svwhilelt_b8(x, window_end_x);
            }
            while(svptest_any(svptrue_b8(), pg));
        },
        input1, input2, output);
    }
}

add_qasymm8_sve2()

void add_qasymm8_sve2	(	const ITensor *	src0,
		const ITensor *	src1,
		ITensor *	dst,
		const ConvertPolicy &	policy,
		const Window &	window
	)

Definition at line 37 of file qasymm8.cpp.

References ARM_COMPUTE_UNUSED, arm_compute::test::validation::b, Window::broadcast_if_dimension_le_one(), Window::DimX, Window::Dimension::end(), arm_compute::execute_window_loop(), ITensor::info(), UniformQuantizationInfo::offset, Iterator::ptr(), ITensorInfo::quantization_info(), UniformQuantizationInfo::scale, Window::set(), Window::Dimension::start(), Window::Dimension::step(), ITensorInfo::tensor_shape(), QuantizationInfo::uniform(), Dimensions< T >::x(), and Window::x().

Referenced by arm_compute::cpu::kernels::can_interpret_inputs_as_1d_array().

{
    ARM_COMPUTE_UNUSED(policy);

    // Create input windows
    Window input1_win = window.broadcast_if_dimension_le_one(src0->info()->tensor_shape());
    Window input2_win = window.broadcast_if_dimension_le_one(src1->info()->tensor_shape());

    // Clear X Dimension on execution window as we handle manually
    Window win = window;
    win.set(Window::DimX, Window::Dimension(0, 1, 1));

    const auto window_start_x        = static_cast<int>(window.x().start());
    const auto window_end_x          = static_cast<int>(window.x().end());
    const bool is_broadcast_across_x = src0->info()->tensor_shape().x() != src1->info()->tensor_shape().x();
    const auto all_true_pg           = svptrue_b8();

    const UniformQuantizationInfo iq1_info = src0->info()->quantization_info().uniform();
    const UniformQuantizationInfo iq2_info = src1->info()->quantization_info().uniform();
    const UniformQuantizationInfo oq_info  = dst->info()->quantization_info().uniform();

    const auto invvscaleo = svdup_n_f32(1.f / oq_info.scale);
    const auto voffseto   = svdup_n_f32(oq_info.offset);

    if(is_broadcast_across_x)
    {
        const bool     is_broadcast_input_2 = input2_win.x().step() == 0;
        Window         broadcast_win        = is_broadcast_input_2 ? input2_win : input1_win;
        Window         non_broadcast_win    = !is_broadcast_input_2 ? input2_win : input1_win;
        const ITensor *broadcast_tensor     = is_broadcast_input_2 ? src1 : src0;
        const ITensor *non_broadcast_tensor = !is_broadcast_input_2 ? src1 : src0;

        const svfloat32_t vscale1  = is_broadcast_input_2 ? svdup_n_f32(iq1_info.scale) : svdup_n_f32(iq2_info.scale);
        const svfloat32_t vscale2  = is_broadcast_input_2 ? svdup_n_f32(iq2_info.scale) : svdup_n_f32(iq1_info.scale);
        const svint32_t   voffset1 = is_broadcast_input_2 ? svdup_n_s32(iq1_info.offset) : svdup_n_s32(iq2_info.offset);
        const svint32_t   voffset2 = is_broadcast_input_2 ? svdup_n_s32(iq2_info.offset) : svdup_n_s32(iq1_info.offset);

        // Clear X Dimension on execution window as we handle manually
        non_broadcast_win.set(Window::DimX, Window::Dimension(0, 1, 1));

        Iterator broadcast_input(broadcast_tensor, broadcast_win);
        Iterator non_broadcast_input(non_broadcast_tensor, non_broadcast_win);
        Iterator output(dst, win);

        execute_window_loop(win, [&](const Coordinates &)
        {
            const auto non_broadcast_input_ptr = reinterpret_cast<const uint8_t *>(non_broadcast_input.ptr());
            const auto output_ptr              = reinterpret_cast<uint8_t *>(output.ptr());

            const uint8_t   broadcast_value     = *reinterpret_cast<const uint8_t *>(broadcast_input.ptr());
            const svuint8_t broadcast_value_vec = svdup_n_u8(broadcast_value);

            int      x  = window_start_x;
            svbool_t pg = svwhilelt_b8(x, window_end_x);

            const auto bf_0 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svreinterpret_s32_u32(svmovlb_u32(svmovlb_u16(broadcast_value_vec))), voffset2)), vscale2);
            const auto bf_1 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svreinterpret_s32_u32(svmovlt_u32(svmovlb_u16(broadcast_value_vec))), voffset2)), vscale2);
            const auto bf_2 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svreinterpret_s32_u32(svmovlb_u32(svmovlt_u16(broadcast_value_vec))), voffset2)), vscale2);
            const auto bf_3 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svreinterpret_s32_u32(svmovlt_u32(svmovlt_u16(broadcast_value_vec))), voffset2)), vscale2);

            do
            {
                const svuint8_t a = svld1_u8(pg, non_broadcast_input_ptr + x);

                const auto af_0 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svreinterpret_s32_u32(svmovlb_u32(svmovlb_u16(a))), voffset1)), vscale1);
                const auto af_1 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svreinterpret_s32_u32(svmovlt_u32(svmovlb_u16(a))), voffset1)), vscale1);
                const auto af_2 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svreinterpret_s32_u32(svmovlb_u32(svmovlt_u16(a))), voffset1)), vscale1);
                const auto af_3 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svreinterpret_s32_u32(svmovlt_u32(svmovlt_u16(a))), voffset1)), vscale1);

                const auto rf_0 = svcvt_u32_f32_z(pg, svmla_f32_z(pg, voffseto, svadd_f32_z(pg, af_0, bf_0), invvscaleo));
                const auto rf_1 = svcvt_u32_f32_z(pg, svmla_f32_z(pg, voffseto, svadd_f32_z(pg, af_1, bf_1), invvscaleo));
                const auto rf_2 = svcvt_u32_f32_z(pg, svmla_f32_z(pg, voffseto, svadd_f32_z(pg, af_2, bf_2), invvscaleo));
                const auto rf_3 = svcvt_u32_f32_z(pg, svmla_f32_z(pg, voffseto, svadd_f32_z(pg, af_3, bf_3), invvscaleo));

                const auto pa = svqxtnt_u32(svqxtnb_u32(rf_0), rf_1);
                const auto pb = svqxtnt_u32(svqxtnb_u32(rf_2), rf_3);

                const auto res = svqxtnt_u16(svqxtnb_u16(pa), pb);
                svst1_u8(pg, output_ptr + x, res);

                x += svcntb();
                pg = svwhilelt_b8(x, window_end_x);
            }
            while(svptest_any(all_true_pg, pg));
        },
        broadcast_input, non_broadcast_input, output);
    }
    else
    {
        // Clear X Dimension on execution window as we handle manually
        input1_win.set(Window::DimX, Window::Dimension(0, 1, 1));
        input2_win.set(Window::DimX, Window::Dimension(0, 1, 1));

        Iterator input1(src0, input1_win);
        Iterator input2(src1, input2_win);
        Iterator output(dst, win);

        const auto vscale1  = svdup_n_f32(iq1_info.scale);
        const auto vscale2  = svdup_n_f32(iq2_info.scale);
        const auto voffset1 = svdup_n_s32(iq1_info.offset);
        const auto voffset2 = svdup_n_s32(iq2_info.offset);

        execute_window_loop(win, [&](const Coordinates &)
        {
            const auto input1_ptr = reinterpret_cast<const uint8_t *>(input1.ptr());
            const auto input2_ptr = reinterpret_cast<const uint8_t *>(input2.ptr());
            const auto output_ptr = reinterpret_cast<uint8_t *>(output.ptr());

            int      x  = window_start_x;
            svbool_t pg = svwhilelt_b8(x, window_end_x);
            do
            {
                const auto a    = svld1_u8(pg, input1_ptr + x);
                const auto b    = svld1_u8(pg, input2_ptr + x);
                const auto af_0 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svreinterpret_s32_u32(svmovlb_u32(svmovlb_u16(a))), voffset1)), vscale1);
                const auto af_1 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svreinterpret_s32_u32(svmovlt_u32(svmovlb_u16(a))), voffset1)), vscale1);
                const auto af_2 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svreinterpret_s32_u32(svmovlb_u32(svmovlt_u16(a))), voffset1)), vscale1);
                const auto af_3 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svreinterpret_s32_u32(svmovlt_u32(svmovlt_u16(a))), voffset1)), vscale1);

                const auto bf_0 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svreinterpret_s32_u32(svmovlb_u32(svmovlb_u16(b))), voffset2)), vscale2);
                const auto bf_1 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svreinterpret_s32_u32(svmovlt_u32(svmovlb_u16(b))), voffset2)), vscale2);
                const auto bf_2 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svreinterpret_s32_u32(svmovlb_u32(svmovlt_u16(b))), voffset2)), vscale2);
                const auto bf_3 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svreinterpret_s32_u32(svmovlt_u32(svmovlt_u16(b))), voffset2)), vscale2);

                const auto rf_0 = svcvt_u32_f32_z(pg, svmla_f32_z(pg, voffseto, svadd_f32_z(pg, af_0, bf_0), invvscaleo));
                const auto rf_1 = svcvt_u32_f32_z(pg, svmla_f32_z(pg, voffseto, svadd_f32_z(pg, af_1, bf_1), invvscaleo));
                const auto rf_2 = svcvt_u32_f32_z(pg, svmla_f32_z(pg, voffseto, svadd_f32_z(pg, af_2, bf_2), invvscaleo));
                const auto rf_3 = svcvt_u32_f32_z(pg, svmla_f32_z(pg, voffseto, svadd_f32_z(pg, af_3, bf_3), invvscaleo));

                const auto pa  = svqxtnt_u32(svqxtnb_u32(rf_0), rf_1);
                const auto pb  = svqxtnt_u32(svqxtnb_u32(rf_2), rf_3);
                const auto res = svqxtnt_u16(svqxtnb_u16(pa), pb);

                svst1_u8(pg, output_ptr + x, res);

                x += svcntb();
                pg = svwhilelt_b8(x, window_end_x);
            }
            while(svptest_any(all_true_pg, pg));
        },
        input1, input2, output);
    }
}

add_qsymm16_neon()

void add_qsymm16_neon	(	const ITensor *	src0,
		const ITensor *	src1,
		ITensor *	dst,
		const ConvertPolicy &	policy,
		const Window &	window
	)

Definition at line 35 of file qsymm16.cpp.

References ARM_COMPUTE_UNUSED, arm_compute::test::validation::b, arm_compute::graph::bfs(), Window::broadcast_if_dimension_le_one(), Window::DimX, Window::Dimension::end(), arm_compute::execute_window_loop(), ITensor::info(), Iterator::ptr(), ITensorInfo::quantization_info(), arm_compute::quantize_qsymm16(), UniformQuantizationInfo::scale, Window::set(), Window::Dimension::start(), Window::Dimension::step(), ITensorInfo::tensor_shape(), QuantizationInfo::uniform(), Dimensions< T >::x(), and Window::x().

Referenced by arm_compute::cpu::kernels::can_interpret_inputs_as_1d_array().

{
    ARM_COMPUTE_UNUSED(policy);

    // Create input windows
    Window input1_win = window.broadcast_if_dimension_le_one(src0->info()->tensor_shape());
    Window input2_win = window.broadcast_if_dimension_le_one(src1->info()->tensor_shape());

    // Clear X Dimension on execution window as we handle manually
    Window win = window;
    win.set(Window::DimX, Window::Dimension(0, 1, 1));

    const int  window_step_x         = 8;
    const auto window_start_x        = static_cast<int>(window.x().start());
    const auto window_end_x          = static_cast<int>(window.x().end());
    const bool is_broadcast_across_x = src0->info()->tensor_shape().x() != src1->info()->tensor_shape().x();

    const UniformQuantizationInfo iq1_info = src0->info()->quantization_info().uniform();
    const UniformQuantizationInfo iq2_info = src1->info()->quantization_info().uniform();
    const UniformQuantizationInfo oq_info  = dst->info()->quantization_info().uniform();

    const float32x4_t vscale1    = vdupq_n_f32(iq1_info.scale);
    const float32x4_t vscale2    = vdupq_n_f32(iq2_info.scale);
    const float32x4_t invvscaleo = vdupq_n_f32(1.f / oq_info.scale);

    if(is_broadcast_across_x)
    {
        const bool                    is_broadcast_input_2 = input2_win.x().step() == 0;
        Window                        broadcast_win        = is_broadcast_input_2 ? input2_win : input1_win;
        Window                        non_broadcast_win    = !is_broadcast_input_2 ? input2_win : input1_win;
        const ITensor                *broadcast_tensor     = is_broadcast_input_2 ? src1 : src0;
        const ITensor                *non_broadcast_tensor = !is_broadcast_input_2 ? src1 : src0;
        const UniformQuantizationInfo broadcast_qinfo      = broadcast_tensor->info()->quantization_info().uniform();
        const UniformQuantizationInfo non_broadcast_qinfo  = non_broadcast_tensor->info()->quantization_info().uniform();

        // Clear X Dimension on execution window as we handle manually
        non_broadcast_win.set(Window::DimX, Window::Dimension(0, 1, 1));

        Iterator broadcast_input(broadcast_tensor, broadcast_win);
        Iterator non_broadcast_input(non_broadcast_tensor, non_broadcast_win);
        Iterator output(dst, win);

        execute_window_loop(win, [&](const Coordinates &)
        {
            const auto non_broadcast_input_ptr = reinterpret_cast<const int16_t *>(non_broadcast_input.ptr());
            const auto output_ptr              = reinterpret_cast<int16_t *>(output.ptr());

            const int16_t   broadcast_value     = *reinterpret_cast<const int16_t *>(broadcast_input.ptr());
            const int16x8_t broadcast_value_vec = vdupq_n_s16(broadcast_value);

            const auto  bf_0 = vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_low_s16(broadcast_value_vec))), vscale2);
            const auto  bf_1 = vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_high_s16(broadcast_value_vec))), vscale2);
            const float bfs  = static_cast<int32_t>(broadcast_value) * broadcast_qinfo.scale;

            // Compute S elements per iteration
            int x = window_start_x;
            for(; x <= (window_end_x - window_step_x); x += window_step_x)
            {
                const int16x8_t a    = vld1q_s16(non_broadcast_input_ptr + x);
                const auto      af_0 = vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_low_s16(a))), vscale1);
                const auto      af_1 = vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_high_s16(a))), vscale1);

                int32x4_t rf_0{};
                int32x4_t rf_1{};
#ifdef __aarch64__
                rf_0 = vcvtnq_s32_f32(vmulq_f32(vaddq_f32(af_0, bf_0), invvscaleo));
                rf_1 = vcvtnq_s32_f32(vmulq_f32(vaddq_f32(af_1, bf_1), invvscaleo));
#else  //__aarch64__
                rf_0 = vcvtq_s32_f32(vmulq_f32(vaddq_f32(af_0, bf_0), invvscaleo));
                rf_1 = vcvtq_s32_f32(vmulq_f32(vaddq_f32(af_1, bf_1), invvscaleo));
#endif //__aarch64__

                const int16x8_t pa = vcombine_s16(vqmovn_s32(rf_0), vqmovn_s32(rf_1));
                vst1q_s16(output_ptr + x, pa);
            }

            // Compute left-over elements
            for(; x < window_end_x; ++x)
            {
                const float afs   = static_cast<int32_t>(*(non_broadcast_input_ptr + x)) * non_broadcast_qinfo.scale;
                *(output_ptr + x) = quantize_qsymm16((afs + bfs), oq_info);
            }
        },
        broadcast_input, non_broadcast_input, output);
    }
    else
    {
        // Clear X Dimension on execution window as we handle manually
        input1_win.set(Window::DimX, Window::Dimension(0, 1, 1));
        input2_win.set(Window::DimX, Window::Dimension(0, 1, 1));

        Iterator input1(src0, input1_win);
        Iterator input2(src1, input2_win);
        Iterator output(dst, win);

        execute_window_loop(win, [&](const Coordinates &)
        {
            const auto input1_ptr = reinterpret_cast<const int16_t *>(input1.ptr());
            const auto input2_ptr = reinterpret_cast<const int16_t *>(input2.ptr());
            const auto output_ptr = reinterpret_cast<int16_t *>(output.ptr());

            // Compute S elements per iteration
            int x = window_start_x;
            for(; x <= (window_end_x - window_step_x); x += window_step_x)
            {
                const int16x8_t a = vld1q_s16(input1_ptr + x);
                const int16x8_t b = vld1q_s16(input2_ptr + x);

                const auto af_0 = vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_low_s16(a))), vscale1);
                const auto af_1 = vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_high_s16(a))), vscale1);
                const auto bf_0 = vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_low_s16(b))), vscale2);
                const auto bf_1 = vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_high_s16(b))), vscale2);

                int32x4_t rf_0{};
                int32x4_t rf_1{};
#ifdef __aarch64__
                rf_0 = vcvtnq_s32_f32(vmulq_f32(vaddq_f32(af_0, bf_0), invvscaleo));
                rf_1 = vcvtnq_s32_f32(vmulq_f32(vaddq_f32(af_1, bf_1), invvscaleo));
#else  //__aarch64__
                rf_0 = vcvtq_s32_f32(vmulq_f32(vaddq_f32(af_0, bf_0), invvscaleo));
                rf_1 = vcvtq_s32_f32(vmulq_f32(vaddq_f32(af_1, bf_1), invvscaleo));
#endif //__aarch64__

                const int16x8_t pa = vcombine_s16(vqmovn_s32(rf_0), vqmovn_s32(rf_1));
                vst1q_s16(output_ptr + x, pa);
            }

            // Compute left-over elements
            for(; x < window_end_x; ++x)
            {
                const float afs   = static_cast<int32_t>((*(input1_ptr + x))) * iq1_info.scale;
                const float bfs   = static_cast<int32_t>((*(input2_ptr + x))) * iq2_info.scale;
                *(output_ptr + x) = quantize_qsymm16((afs + bfs), dst->info()->quantization_info());
            }
        },
        input1, input2, output);
    }
}

add_qsymm16_sve2()

void add_qsymm16_sve2	(	const ITensor *	src0,
		const ITensor *	src1,
		ITensor *	dst,
		const ConvertPolicy &	policy,
		const Window &	window
	)

Definition at line 37 of file qsymm16.cpp.

References ARM_COMPUTE_UNUSED, arm_compute::test::validation::b, Window::broadcast_if_dimension_le_one(), Window::DimX, Window::Dimension::end(), arm_compute::execute_window_loop(), ITensor::info(), Iterator::ptr(), ITensorInfo::quantization_info(), UniformQuantizationInfo::scale, Window::set(), Window::Dimension::start(), Window::Dimension::step(), ITensorInfo::tensor_shape(), QuantizationInfo::uniform(), Dimensions< T >::x(), and Window::x().

Referenced by arm_compute::cpu::kernels::can_interpret_inputs_as_1d_array().

{
    ARM_COMPUTE_UNUSED(policy);

    // Create input windows
    Window input1_win = window.broadcast_if_dimension_le_one(src0->info()->tensor_shape());
    Window input2_win = window.broadcast_if_dimension_le_one(src1->info()->tensor_shape());

    // Clear X Dimension on execution window as we handle manually
    Window win = window;
    win.set(Window::DimX, Window::Dimension(0, 1, 1));

    const auto window_start_x        = static_cast<int>(window.x().start());
    const auto window_end_x          = static_cast<int>(window.x().end());
    const bool is_broadcast_across_x = src0->info()->tensor_shape().x() != src1->info()->tensor_shape().x();

    const UniformQuantizationInfo iq1_info = src0->info()->quantization_info().uniform();
    const UniformQuantizationInfo iq2_info = src1->info()->quantization_info().uniform();
    const UniformQuantizationInfo oq_info  = dst->info()->quantization_info().uniform();

    const auto vscale1     = svdup_n_f32(iq1_info.scale);
    const auto vscale2     = svdup_n_f32(iq2_info.scale);
    const auto invvscaleo  = svdup_n_f32(1.f / oq_info.scale);
    const auto all_true_pg = svptrue_b16();

    if(is_broadcast_across_x)
    {
        const bool     is_broadcast_input_2 = input2_win.x().step() == 0;
        Window         broadcast_win        = is_broadcast_input_2 ? input2_win : input1_win;
        Window         non_broadcast_win    = !is_broadcast_input_2 ? input2_win : input1_win;
        const ITensor *broadcast_tensor     = is_broadcast_input_2 ? src1 : src0;
        const ITensor *non_broadcast_tensor = !is_broadcast_input_2 ? src1 : src0;

        // Clear X Dimension on execution window as we handle manually
        non_broadcast_win.set(Window::DimX, Window::Dimension(0, 1, 1));

        Iterator broadcast_input(broadcast_tensor, broadcast_win);
        Iterator non_broadcast_input(non_broadcast_tensor, non_broadcast_win);
        Iterator output(dst, win);

        execute_window_loop(win, [&](const Coordinates &)
        {
            const auto non_broadcast_input_ptr = reinterpret_cast<const int16_t *>(non_broadcast_input.ptr());
            const auto output_ptr              = reinterpret_cast<int16_t *>(output.ptr());

            const int16_t broadcast_value     = *reinterpret_cast<const int16_t *>(broadcast_input.ptr());
            const auto    broadcast_value_vec = svdup_n_s16(broadcast_value);

            int      x  = window_start_x;
            svbool_t pg = svwhilelt_b16(x, window_end_x);

            const auto bf_0 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svmovlb_s32(broadcast_value_vec)), vscale2);
            const auto bf_1 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svmovlt_s32(broadcast_value_vec)), vscale2);

            do
            {
                const auto a    = svld1_s16(pg, non_broadcast_input_ptr + x);
                const auto af_0 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svmovlb_s32(a)), vscale1);
                const auto af_1 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svmovlt_s32(a)), vscale1);

                const auto rf_0 = svcvt_s32_f32_z(pg, svmul_f32_z(pg, svadd_f32_z(pg, af_0, bf_0), invvscaleo));
                const auto rf_1 = svcvt_s32_f32_z(pg, svmul_f32_z(pg, svadd_f32_z(pg, af_1, bf_1), invvscaleo));

                const auto res = svqxtnt_s32(svqxtnb_s32(rf_0), rf_1);

                svst1_s16(pg, output_ptr + x, res);

                x += svcnth();
                pg = svwhilelt_b16(x, window_end_x);
            }
            while(svptest_any(all_true_pg, pg));
        },
        broadcast_input, non_broadcast_input, output);
    }
    else
    {
        // Clear X Dimension on execution window as we handle manually
        input1_win.set(Window::DimX, Window::Dimension(0, 1, 1));
        input2_win.set(Window::DimX, Window::Dimension(0, 1, 1));

        Iterator input1(src0, input1_win);
        Iterator input2(src1, input2_win);
        Iterator output(dst, win);

        execute_window_loop(win, [&](const Coordinates &)
        {
            const auto input1_ptr = reinterpret_cast<const int16_t *>(input1.ptr());
            const auto input2_ptr = reinterpret_cast<const int16_t *>(input2.ptr());
            const auto output_ptr = reinterpret_cast<int16_t *>(output.ptr());

            int      x  = window_start_x;
            svbool_t pg = svwhilelt_b16(x, window_end_x);
            do
            {
                auto a = svld1_s16(pg, input1_ptr + x);
                auto b = svld1_s16(pg, input2_ptr + x);

                const auto af_0 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svmovlb_s32(a)), vscale1);
                const auto af_1 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svmovlt_s32(a)), vscale1);

                const auto bf_0 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svmovlb_s32(b)), vscale2);
                const auto bf_1 = svmul_f32_z(pg, svcvt_f32_s32_z(pg, svmovlt_s32(b)), vscale2);

                const auto rf_0 = svcvt_s32_f32_z(pg, svmul_f32_z(pg, svadd_f32_z(pg, af_0, bf_0), invvscaleo));
                const auto rf_1 = svcvt_s32_f32_z(pg, svmul_f32_z(pg, svadd_f32_z(pg, af_1, bf_1), invvscaleo));

                const auto res = svqxtnt_s32(svqxtnb_s32(rf_0), rf_1);
                svst1_s16(pg, output_ptr + x, res);

                x += svcnth();
                pg = svwhilelt_b16(x, window_end_x);
            }
            while(svptest_any(all_true_pg, pg));
        },
        input1, input2, output);
    }
}

add_s16_neon()

void add_s16_neon	(	const ITensor *	src0,
		const ITensor *	src1,
		ITensor *	dst,
		const ConvertPolicy &	policy,
		const Window &	window
	)

Definition at line 36 of file integer.cpp.

References add_same_neon< int16_t >(), and arm_compute::test::validation::dst.

Referenced by arm_compute::cpu::kernels::can_interpret_inputs_as_1d_array().

{
    return add_same_neon<int16_t>(src0, src1, dst, policy, window);
}

add_s16_neon_as_1d_array()

void add_s16_neon_as_1d_array	(	const ITensor *	src0,
		const ITensor *	src1,
		ITensor *	dst,
		const ConvertPolicy &	policy,
		const Window &	window
	)

Definition at line 51 of file integer.cpp.

References add_same_neon_as_1d_array< int16_t >(), and arm_compute::test::validation::dst.

Referenced by arm_compute::cpu::kernels::can_interpret_inputs_as_1d_array().

{
    return add_same_neon_as_1d_array<int16_t>(src0, src1, dst, policy, window);
}

add_s16_sve()

void add_s16_sve	(	const ITensor *	src0,
		const ITensor *	src1,
		ITensor *	dst,
		const ConvertPolicy &	policy,
		const Window &	window
	)

Definition at line 38 of file integer.cpp.

References add_same_sve< int16_t >(), and arm_compute::test::validation::dst.

Referenced by arm_compute::cpu::kernels::can_interpret_inputs_as_1d_array().

{
    return add_same_sve<int16_t>(src0, src1, dst, policy, window);
}

add_s32_neon()

void add_s32_neon	(	const ITensor *	src0,
		const ITensor *	src1,
		ITensor *	dst,
		const ConvertPolicy &	policy,
		const Window &	window
	)

Definition at line 41 of file integer.cpp.

References add_same_neon< int32_t >(), and arm_compute::test::validation::dst.

Referenced by arm_compute::cpu::kernels::can_interpret_inputs_as_1d_array().

{
    return add_same_neon<int32_t>(src0, src1, dst, policy, window);
}

add_s32_neon_as_1d_array()

void add_s32_neon_as_1d_array	(	const ITensor *	src0,
		const ITensor *	src1,
		ITensor *	dst,
		const ConvertPolicy &	policy,
		const Window &	window
	)

Definition at line 56 of file integer.cpp.

References add_same_neon_as_1d_array< int32_t >(), and arm_compute::test::validation::dst.

Referenced by arm_compute::cpu::kernels::can_interpret_inputs_as_1d_array().

{
    return add_same_neon_as_1d_array<int32_t>(src0, src1, dst, policy, window);
}

add_s32_sve()

void add_s32_sve	(	const ITensor *	src0,
		const ITensor *	src1,
		ITensor *	dst,
		const ConvertPolicy &	policy,
		const Window &	window
	)

Definition at line 43 of file integer.cpp.

References add_same_sve< int32_t >(), and arm_compute::test::validation::dst.

Referenced by arm_compute::cpu::kernels::can_interpret_inputs_as_1d_array().

{
    return add_same_sve<int32_t>(src0, src1, dst, policy, window);
}

add_same_neon()

void add_same_neon	(	const ITensor *	src0,
		const ITensor *	src1,
		ITensor *	dst,
		const ConvertPolicy &	policy,
		const Window &	window
	)

SIMD vector tag type.

Definition at line 34 of file impl.cpp.

References arm_compute::wrapper::add_sat(), Window::broadcast_if_dimension_le_one(), Window::DimX, Window::Dimension::end(), arm_compute::execute_window_loop(), ITensor::info(), Iterator::ptr(), arm_compute::SATURATE, Window::set(), Window::Dimension::start(), Window::Dimension::step(), ITensorInfo::tensor_shape(), arm_compute::wrapper::vadd(), arm_compute::wrapper::vdup_n(), arm_compute::wrapper::vloadq(), arm_compute::wrapper::vqadd(), arm_compute::wrapper::vstore(), Dimensions< T >::x(), and Window::x().

{
    /** SIMD vector tag type. */
    using ExactTagType = typename wrapper::traits::neon_bitvector_tag_t<ScalarType, wrapper::traits::BitWidth::W128>;

    // Create input windows
    Window input1_win = window.broadcast_if_dimension_le_one(src0->info()->tensor_shape());
    Window input2_win = window.broadcast_if_dimension_le_one(src1->info()->tensor_shape());

    // Clear X Dimension on execution window as we handle manually
    Window win = window;
    win.set(Window::DimX, Window::Dimension(0, 1, 1));

    constexpr int window_step_x         = 16 / sizeof(ScalarType);
    const auto    window_start_x        = static_cast<int>(window.x().start());
    const auto    window_end_x          = static_cast<int>(window.x().end());
    const bool    is_broadcast_across_x = src0->info()->tensor_shape().x() != src1->info()->tensor_shape().x();

    if(is_broadcast_across_x)
    {
        const bool     is_broadcast_input_2 = input2_win.x().step() == 0;
        Window         broadcast_win        = is_broadcast_input_2 ? input2_win : input1_win;
        Window         non_broadcast_win    = !is_broadcast_input_2 ? input2_win : input1_win;
        const ITensor *broadcast_tensor     = is_broadcast_input_2 ? src1 : src0;
        const ITensor *non_broadcast_tensor = !is_broadcast_input_2 ? src1 : src0;

        // Clear X Dimension on execution window as we handle manually
        non_broadcast_win.set(Window::DimX, Window::Dimension(0, 1, 1));

        Iterator broadcast_input(broadcast_tensor, broadcast_win);
        Iterator non_broadcast_input(non_broadcast_tensor, non_broadcast_win);
        Iterator output(dst, win);

        execute_window_loop(win, [&](const Coordinates &)
        {
            const auto non_broadcast_input_ptr = reinterpret_cast<const ScalarType *>(non_broadcast_input.ptr());
            const auto output_ptr              = reinterpret_cast<ScalarType *>(output.ptr());

            const ScalarType broadcast_value     = *reinterpret_cast<const ScalarType *>(broadcast_input.ptr());
            const auto       broadcast_value_vec = wrapper::vdup_n(broadcast_value, ExactTagType{});

            // Compute S elements per iteration
            int x = window_start_x;
            for(; x <= (window_end_x - window_step_x); x += window_step_x)
            {
                const auto non_broadcast_v = wrapper::vloadq(non_broadcast_input_ptr + x);
                const auto res             = (policy == ConvertPolicy::SATURATE) ? wrapper::vqadd(broadcast_value_vec, non_broadcast_v) : wrapper::vadd(broadcast_value_vec, non_broadcast_v);
                wrapper::vstore(output_ptr + x, res);
            }

            // Compute left-over elements
            for(; x < window_end_x; ++x)
            {
                const auto non_broadcast_v = *(non_broadcast_input_ptr + x);
                *(output_ptr + x)          = (policy == ConvertPolicy::SATURATE) ? wrapper::add_sat(broadcast_value, non_broadcast_v) : broadcast_value + non_broadcast_v;
            }
        },
        broadcast_input, non_broadcast_input, output);
    }
    else
    {
        // Clear X Dimension on execution window as we handle manually
        input1_win.set(Window::DimX, Window::Dimension(0, 1, 1));
        input2_win.set(Window::DimX, Window::Dimension(0, 1, 1));

        Iterator input1(src0, input1_win);
        Iterator input2(src1, input2_win);
        Iterator output(dst, win);

        execute_window_loop(win, [&](const Coordinates &)
        {
            const auto input1_ptr = reinterpret_cast<const ScalarType *>(input1.ptr());
            const auto input2_ptr = reinterpret_cast<const ScalarType *>(input2.ptr());
            const auto output_ptr = reinterpret_cast<ScalarType *>(output.ptr());

            // Compute S elements per iteration
            int x = window_start_x;
            for(; x <= (window_end_x - window_step_x); x += window_step_x)
            {
                const auto val1 = wrapper::vloadq(input1_ptr + x);
                const auto val2 = wrapper::vloadq(input2_ptr + x);
                const auto res  = (policy == ConvertPolicy::SATURATE) ? wrapper::vqadd(val1, val2) : wrapper::vadd(val1, val2);
                wrapper::vstore(output_ptr + x, res);
            }

            // Compute left-over elements
            for(; x < window_end_x; ++x)
            {
                const auto val1   = *(input1_ptr + x);
                const auto val2   = *(input2_ptr + x);
                *(output_ptr + x) = (policy == ConvertPolicy::SATURATE) ? wrapper::add_sat(val1, val2) : val1 + val2;
            }
        },
        input1, input2, output);
    }
}

add_same_neon< float >()

template void arm_compute::cpu::add_same_neon< float >	(	const ITensor *	src0,
		const ITensor *	src1,
		ITensor *	dst,
		const ConvertPolicy &	policy,
		const Window &	window
	)

Referenced by add_fp32_neon(), and add_same_neon_as_1d_array().

add_same_neon< int16_t >()

template void arm_compute::cpu::add_same_neon< int16_t >	(	const ITensor *	src0,
		const ITensor *	src1,
		ITensor *	dst,
		const ConvertPolicy &	policy,
		const Window &	window
	)

Referenced by add_s16_neon(), and add_same_neon_as_1d_array().

add_same_neon< int32_t >()

template void arm_compute::cpu::add_same_neon< int32_t >	(	const ITensor *	src0,
		const ITensor *	src1,
		ITensor *	dst,
		const ConvertPolicy &	policy,
		const Window &	window
	)

Referenced by add_s32_neon(), and add_same_neon_as_1d_array().

add_same_neon< uint8_t >()

template void arm_compute::cpu::add_same_neon< uint8_t >	(	const ITensor *	src0,
		const ITensor *	src1,
		ITensor *	dst,
		const ConvertPolicy &	policy,
		const Window &	window
	)

Referenced by add_same_neon_as_1d_array(), and add_u8_neon().

add_same_neon_as_1d_array()

void add_same_neon_as_1d_array	(	const ITensor *	src0,
		const ITensor *	src1,
		ITensor *	dst,
		const ConvertPolicy &	policy,
		const Window &	window
	)

Definition at line 132 of file impl.cpp.

References add_same_neon< float >(), add_same_neon< int16_t >(), add_same_neon< int32_t >(), add_same_neon< uint8_t >(), add_same_neon_as_1d_array< float >(), add_same_neon_as_1d_array< int16_t >(), add_same_neon_as_1d_array< int32_t >(), add_same_neon_as_1d_array< uint8_t >(), arm_compute::wrapper::add_sat(), ITensor::buffer(), arm_compute::test::validation::dst, Window::Dimension::end(), arm_compute::SATURATE, Window::Dimension::start(), arm_compute::wrapper::vadd(), arm_compute::wrapper::vloadq(), arm_compute::wrapper::vqadd(), arm_compute::wrapper::vstore(), and Window::x().

{
    const ScalarType *src0_ptr = reinterpret_cast<const ScalarType *>(src0->buffer());
    const ScalarType *src1_ptr = reinterpret_cast<const ScalarType *>(src1->buffer());
    ScalarType       *dst_ptr  = reinterpret_cast<ScalarType *>(dst->buffer());

    constexpr int window_step_x  = 16 / sizeof(ScalarType);
    const auto    window_start_x = static_cast<int>(window.x().start());
    const auto    window_end_x   = static_cast<int>(window.x().end());

    int x = window_start_x;
    for(; x <= (window_end_x - window_step_x); x += window_step_x)
    {
        const auto val1 = wrapper::vloadq(src0_ptr + x);
        const auto val2 = wrapper::vloadq(src1_ptr + x);
        const auto res  = (policy == ConvertPolicy::SATURATE) ? wrapper::vqadd(val1, val2) : wrapper::vadd(val1, val2);
        wrapper::vstore(dst_ptr + x, res);
    }

    // Compute left-over elements
    for(; x < window_end_x; ++x)
    {
        const auto val1 = *(src0_ptr + x);
        const auto val2 = *(src1_ptr + x);
        *(dst_ptr + x)  = (policy == ConvertPolicy::SATURATE) ? wrapper::add_sat(val1, val2) : val1 + val2;
    }
}

add_same_neon_as_1d_array< float >()

template void arm_compute::cpu::add_same_neon_as_1d_array< float >	(	const ITensor *	src0,
		const ITensor *	src1,
		ITensor *	dst,
		const ConvertPolicy &	policy,
		const Window &	window
	)

Referenced by add_fp32_neon_as_1d_array(), and add_same_neon_as_1d_array().

add_same_neon_as_1d_array< int16_t >()

template void arm_compute::cpu::add_same_neon_as_1d_array< int16_t >	(	const ITensor *	src0,
		const ITensor *	src1,
		ITensor *	dst,
		const ConvertPolicy &	policy,
		const Window &	window
	)

Referenced by add_s16_neon_as_1d_array(), and add_same_neon_as_1d_array().

add_same_neon_as_1d_array< int32_t >()

template void arm_compute::cpu::add_same_neon_as_1d_array< int32_t >	(	const ITensor *	src0,
		const ITensor *	src1,
		ITensor *	dst,
		const ConvertPolicy &	policy,
		const Window &	window
	)

Referenced by add_s32_neon_as_1d_array(), and add_same_neon_as_1d_array().

add_same_neon_as_1d_array< uint8_t >()

template void arm_compute::cpu::add_same_neon_as_1d_array< uint8_t >	(	const ITensor *	src0,
		const ITensor *	src1,
		ITensor *	dst,
		const ConvertPolicy &	policy,
		const Window &	window
	)

Referenced by add_same_neon_as_1d_array(), and add_u8_neon_as_1d_array().

add_same_sve()

void add_same_sve	(	const ITensor *	src0,
		const ITensor *	src1,
		ITensor *	dst,
		const ConvertPolicy &	policy,
		const Window &	window
	)

Definition at line 36 of file impl.cpp.

References add_same_sve< float >(), add_same_sve< int16_t >(), add_same_sve< int32_t >(), add_same_sve< uint8_t >(), Window::broadcast_if_dimension_le_one(), Window::DimX, arm_compute::test::validation::dst, Window::Dimension::end(), arm_compute::execute_window_loop(), ITensor::info(), Iterator::ptr(), arm_compute::SATURATE, Window::set(), Window::Dimension::start(), Window::Dimension::step(), ITensorInfo::tensor_shape(), Dimensions< T >::x(), and Window::x().

{
    const auto all_true_pg           = wrapper::svptrue<ScalarType>();
    const auto window_start_x        = static_cast<int>(window.x().start());
    const auto window_end_x          = static_cast<int>(window.x().end());
    const bool is_broadcast_across_x = src0->info()->tensor_shape().x() != src1->info()->tensor_shape().x();
    const bool is_sat                = (policy == ConvertPolicy::SATURATE);

    // Clear X Dimension on execution window as we handle manually
    Window win = window;
    win.set(Window::DimX, Window::Dimension(0, 1, 1));

    // Create input windows
    Window input1_win = window.broadcast_if_dimension_le_one(src0->info()->tensor_shape());
    Window input2_win = window.broadcast_if_dimension_le_one(src1->info()->tensor_shape());

    Iterator input1(src0, window.broadcast_if_dimension_le_one(src0->info()->tensor_shape()));
    Iterator input2(src1, window.broadcast_if_dimension_le_one(src1->info()->tensor_shape()));
    Iterator output(dst, window);

    if(is_broadcast_across_x)
    {
        const bool     is_broadcast_input_2 = input2_win.x().step() == 0;
        Window         broadcast_win        = is_broadcast_input_2 ? input2_win : input1_win;
        Window         non_broadcast_win    = !is_broadcast_input_2 ? input2_win : input1_win;
        const ITensor *broadcast_tensor     = is_broadcast_input_2 ? src1 : src0;
        const ITensor *non_broadcast_tensor = !is_broadcast_input_2 ? src1 : src0;

        // Clear X Dimension on execution window as we handle manually
        non_broadcast_win.set(Window::DimX, Window::Dimension(0, 1, 1));

        Iterator broadcast_input(broadcast_tensor, broadcast_win);
        Iterator non_broadcast_input(non_broadcast_tensor, non_broadcast_win);
        Iterator output(dst, win);

        execute_window_loop(win, [&](const Coordinates &)
        {
            const auto non_broadcast_input_ptr = reinterpret_cast<const ScalarType *>(non_broadcast_input.ptr());
            const auto output_ptr              = reinterpret_cast<ScalarType *>(output.ptr());

            const ScalarType broadcast_value     = *reinterpret_cast<const ScalarType *>(broadcast_input.ptr());
            const auto       broadcast_value_vec = wrapper::svdup_n(broadcast_value);

            int      x  = window_start_x;
            svbool_t pg = wrapper::svwhilelt<ScalarType>(x, window_end_x);
            do
            {
                const auto non_broadcast_v = svld1(pg, non_broadcast_input_ptr + x);
                auto       res             = is_sat ? wrapper::svqadd(broadcast_value_vec, non_broadcast_v) : svadd_z(pg, broadcast_value_vec, non_broadcast_v);
                svst1(pg, output_ptr + x, res);

                x += wrapper::svcnt<ScalarType>();
                pg = wrapper::svwhilelt<ScalarType>(x, window_end_x);
            }
            while(svptest_any(all_true_pg, pg));
        },
        broadcast_input, non_broadcast_input, output);
    }
    else
    {
        // Clear X Dimension on execution window as we handle manually
        input1_win.set(Window::DimX, Window::Dimension(0, 1, 1));
        input2_win.set(Window::DimX, Window::Dimension(0, 1, 1));

        Iterator input1(src0, input1_win);
        Iterator input2(src1, input2_win);
        Iterator output(dst, win);

        execute_window_loop(win, [&](const Coordinates &)
        {
            const auto input1_ptr = reinterpret_cast<const ScalarType *>(input1.ptr());
            const auto input2_ptr = reinterpret_cast<const ScalarType *>(input2.ptr());
            const auto output_ptr = reinterpret_cast<ScalarType *>(output.ptr());

            int      x  = window_start_x;
            svbool_t pg = wrapper::svwhilelt<ScalarType>(x, window_end_x);
            do
            {
                const auto val1 = svld1(pg, input1_ptr + x);
                const auto val2 = svld1(pg, input2_ptr + x);
                const auto res  = is_sat ? wrapper::svqadd(val1, val2) : svadd_z(pg, val1, val2);
                svst1(pg, output_ptr + x, res);

                x += wrapper::svcnt<ScalarType>();
                pg = wrapper::svwhilelt<ScalarType>(x, window_end_x);
            }
            while(svptest_any(all_true_pg, pg));
        },
        input1, input2, output);
    }
}

add_same_sve< float >()

template void arm_compute::cpu::add_same_sve< float >	(	const ITensor *	src0,
		const ITensor *	src1,
		ITensor *	dst,
		const ConvertPolicy &	policy,
		const Window &	window
	)

Referenced by add_fp32_sve(), and add_same_sve().

add_same_sve< int16_t >()

template void arm_compute::cpu::add_same_sve< int16_t >	(	const ITensor *	src0,
		const ITensor *	src1,
		ITensor *	dst,
		const ConvertPolicy &	policy,
		const Window &	window
	)

Referenced by add_s16_sve(), and add_same_sve().

add_same_sve< int32_t >()

template void arm_compute::cpu::add_same_sve< int32_t >	(	const ITensor *	src0,
		const ITensor *	src1,
		ITensor *	dst,
		const ConvertPolicy &	policy,
		const Window &	window
	)

Referenced by add_s32_sve(), and add_same_sve().

add_same_sve< uint8_t >()

template void arm_compute::cpu::add_same_sve< uint8_t >	(	const ITensor *	src0,
		const ITensor *	src1,
		ITensor *	dst,
		const ConvertPolicy &	policy,
		const Window &	window
	)

Referenced by add_same_sve(), and add_u8_sve().

add_u8_neon()

void add_u8_neon	(	const ITensor *	src0,
		const ITensor *	src1,
		ITensor *	dst,
		const ConvertPolicy &	policy,
		const Window &	window
	)

Definition at line 31 of file integer.cpp.

References add_same_neon< uint8_t >(), and arm_compute::test::validation::dst.

Referenced by arm_compute::cpu::kernels::can_interpret_inputs_as_1d_array().

{
    return add_same_neon<uint8_t>(src0, src1, dst, policy, window);
}

add_u8_neon_as_1d_array()

void add_u8_neon_as_1d_array	(	const ITensor *	src0,
		const ITensor *	src1,
		ITensor *	dst,
		const ConvertPolicy &	policy,
		const Window &	window
	)

Definition at line 46 of file integer.cpp.

References add_same_neon_as_1d_array< uint8_t >(), and arm_compute::test::validation::dst.

Referenced by arm_compute::cpu::kernels::can_interpret_inputs_as_1d_array().

{
    return add_same_neon_as_1d_array<uint8_t>(src0, src1, dst, policy, window);
}

add_u8_sve()

void add_u8_sve	(	const ITensor *	src0,
		const ITensor *	src1,
		ITensor *	dst,
		const ConvertPolicy &	policy,
		const Window &	window
	)

Definition at line 33 of file integer.cpp.

References add_same_sve< uint8_t >(), and arm_compute::test::validation::dst.

Referenced by arm_compute::cpu::kernels::can_interpret_inputs_as_1d_array().

{
    return add_same_sve<uint8_t>(src0, src1, dst, policy, window);
}

avg_poolingMxNxD_q8_neon_ndhwc()

void arm_compute::cpu::avg_poolingMxNxD_q8_neon_ndhwc	(	const ITensor *	src,
		ITensor *	dst0,
		Pooling3dLayerInfo &	pool_info,
		const Window &	window_out,
		const int	window_step_x
	)

Definition at line 38 of file quantized.h.

References Padding3D::back, Padding3D::bottom, ITensor::buffer(), Size3D::depth, ITensorInfo::dimension(), Pooling3dLayerInfo::exclude_padding, arm_compute::execute_window_loop(), Padding3D::front, Size3D::height, ITensor::info(), Pooling3dLayerInfo::is_global_pooling, Padding3D::left, UniformQuantizationInfo::offset, ITensorInfo::offset_first_element_in_bytes(), Pooling3dLayerInfo::padding, Pooling3dLayerInfo::pool_size, Iterator::ptr(), ITensorInfo::quantization_info(), Padding3D::right, UniformQuantizationInfo::scale, arm_compute::test::validation::scale, Pooling3dLayerInfo::stride, ITensorInfo::strides_in_bytes(), ITensorInfo::tensor_shape(), Padding3D::top, QuantizationInfo::uniform(), arm_compute::wrapper::vadd(), arm_compute::wrapper::vcombine(), arm_compute::wrapper::vdup_n(), arm_compute::wrapper::vgethigh(), arm_compute::wrapper::vgetlow(), arm_compute::wrapper::vloadq(), arm_compute::wrapper::vmla(), arm_compute::wrapper::vmovl(), arm_compute::wrapper::vmovn(), arm_compute::wrapper::vstore(), Size3D::width, Dimensions< T >::y(), and Dimensions< T >::z().

{
    using q8x8_t  = typename wrapper::traits::neon_vector<T, 8>::type;
    using q8x16_t = typename wrapper::traits::neon_vector<T, 16>::type;
    using q16_t   = typename wrapper::traits::promote_t<T>;
    using q16x8_t = typename wrapper::traits::neon_vector<q16_t, 8>::type;
    using q32_t   = typename wrapper::traits::promote_t<q16_t>;
    using q32x4_t = typename wrapper::traits::neon_vector<q32_t, 4>::type;

    int pool_stride_x = static_cast<int>(pool_info.stride.width);
    int pool_stride_y = static_cast<int>(pool_info.stride.height);
    int pool_stride_z = static_cast<int>(pool_info.stride.depth);

    const int pool_size_x = pool_info.is_global_pooling ? src->info()->tensor_shape().y() : pool_info.pool_size.width;
    const int pool_size_y = pool_info.is_global_pooling ? src->info()->tensor_shape().z() : pool_info.pool_size.height;
    const int pool_size_z = pool_info.is_global_pooling ? src->info()->tensor_shape()[3] : pool_info.pool_size.depth;

    const int pool_pad_top    = static_cast<int>(pool_info.padding.top);
    const int pool_pad_bottom = static_cast<int>(pool_info.padding.bottom);
    const int pool_pad_left   = static_cast<int>(pool_info.padding.left);
    const int pool_pad_right  = static_cast<int>(pool_info.padding.right);
    const int pool_pad_front  = static_cast<int>(pool_info.padding.front);
    const int pool_pad_back   = static_cast<int>(pool_info.padding.back);

    const int upper_bound_w = src->info()->dimension(1) + (pool_info.exclude_padding ? 0 : pool_pad_right);
    const int upper_bound_h = src->info()->dimension(2) + (pool_info.exclude_padding ? 0 : pool_pad_bottom);
    const int upper_bound_d = src->info()->dimension(3) + (pool_info.exclude_padding ? 0 : pool_pad_back);

    const int input_dim_c = src->info()->dimension(0);
    const int input_dim_w = src->info()->dimension(1);
    const int input_dim_h = src->info()->dimension(2);
    const int input_dim_d = src->info()->dimension(3);

    const int y_stride = static_cast<int>(src->info()->strides_in_bytes().y());
    const int z_stride = static_cast<int>(src->info()->strides_in_bytes().z());
    const int w_stride = static_cast<int>(src->info()->strides_in_bytes()[3]);
    const int n_stride = static_cast<int>(src->info()->strides_in_bytes()[4]);

    const uint8_t *in_ptr_start = src->buffer() + src->info()->offset_first_element_in_bytes();

    const int window_end_x   = input_dim_c;
    const int window_start_x = 0;

    Iterator out(dst0, window_out);

    const float32x4_t             half_scale_v = vdupq_n_f32(0.5f);
    const UniformQuantizationInfo src_qinfo    = src->info()->quantization_info().uniform();
    const UniformQuantizationInfo dst_qinfo    = dst0->info()->quantization_info().uniform();

    const float quant_rescale = dst_qinfo.scale / src_qinfo.scale;
    // "new_offset" doesn't have to consider the "half_scale_v" in its computation
    // With a requantization performed in a single step there won't be uncertainties introduced
    const int32_t new_offset = dst_qinfo.offset - static_cast<int32_t>(static_cast<float>(src_qinfo.offset) / quant_rescale);

    execute_window_loop(window_out, [&](const Coordinates & id)
    {
        // Computing the theoretical input starting/ending points
        const int in_idx_width  = static_cast<int>(id.y()) * pool_stride_x - pool_pad_left;
        const int in_idx_height = static_cast<int>(id.z()) * pool_stride_y - pool_pad_top;
        const int in_idx_depth  = static_cast<int>(id[3]) * pool_stride_z - pool_pad_front;

        const int pool_start_x = std::max(0, -in_idx_width);
        const int pool_end_x_t = std::min(input_dim_w + pool_pad_left - in_idx_width, pool_size_x);
        const int pool_start_y = std::max(0, -in_idx_height);
        const int pool_end_y_t = std::min(input_dim_h + pool_pad_top - in_idx_height, pool_size_y);

        const int pool_start_z = std::max(0, -in_idx_depth);
        const int pool_end_z_t = std::min(input_dim_d + pool_pad_front - in_idx_depth, pool_size_z);

        // The end of width to consider in calculation should exclude PAD_X, PAD_Y and PAD_Z
        const int pool_end_x = std::min(pool_end_x_t, input_dim_w - in_idx_width);
        const int pool_end_y = std::min(pool_end_y_t, input_dim_h - in_idx_height);
        const int pool_end_z = std::min(pool_end_z_t, input_dim_d - in_idx_depth);

        // Calculate scale
        const float scale = calculate_avg_scale_pool3d(pool_info.exclude_padding, id, pool_size_x, pool_size_y, pool_size_z, upper_bound_w, upper_bound_h, upper_bound_d, pool_pad_left,
                                                       pool_pad_top, pool_pad_front, pool_stride_x, pool_stride_y, pool_stride_z);

        const uint8_t *in_ptr_n = in_ptr_start + id[4] * n_stride;

        int x_off = window_start_x;

        for(; x_off <= (window_end_x - window_step_x); x_off += window_step_x) // C
        {
            q32x4_t vres1 = wrapper::vdup_n(static_cast<q32_t>(0.f), wrapper::traits::vector_128_tag{});
            q32x4_t vres2 = wrapper::vdup_n(static_cast<q32_t>(0.f), wrapper::traits::vector_128_tag{});
            q32x4_t vres3 = wrapper::vdup_n(static_cast<q32_t>(0.f), wrapper::traits::vector_128_tag{});
            q32x4_t vres4 = wrapper::vdup_n(static_cast<q32_t>(0.f), wrapper::traits::vector_128_tag{});

            // Perform pooling
            for(int z = pool_start_z; z < pool_end_z; ++z)
            {
                const uint8_t *in_ptr_z = in_ptr_n + (z + in_idx_depth) * w_stride;
                for(int y = pool_start_y; y < pool_end_y; ++y)
                {
                    const uint8_t *in_ptr_y = in_ptr_z + (y + in_idx_height) * z_stride;
                    for(int x = pool_start_x; x < pool_end_x; ++x)
                    {
                        const uint8_t *in_ptr_x = in_ptr_y + (x + in_idx_width) * y_stride;
                        const q8x16_t  data     = wrapper::vloadq(reinterpret_cast<const T *>(in_ptr_x) + x_off);

                        const q16x8_t data_q16  = wrapper::vmovl(wrapper::vgetlow(data));
                        const q16x8_t data2_q16 = wrapper::vmovl(wrapper::vgethigh(data));
                        vres1                   = wrapper::vadd(vres1, wrapper::vmovl(wrapper::vgetlow(data_q16)));
                        vres2                   = wrapper::vadd(vres2, wrapper::vmovl(wrapper::vgethigh(data_q16)));
                        vres3                   = wrapper::vadd(vres3, wrapper::vmovl(wrapper::vgetlow(data2_q16)));
                        vres4                   = wrapper::vadd(vres4, wrapper::vmovl(wrapper::vgethigh(data2_q16)));
                    }
                }
            }

            if(src_qinfo != dst_qinfo)
            {
                const float32x4x4_t vres =
                {
                    {
                        vcvtq_f32_q32(vres1),
                        vcvtq_f32_q32(vres2),
                        vcvtq_f32_q32(vres3),
                        vcvtq_f32_q32(vres4),
                    }
                };
                const auto requantized_dst = vrequantize_pooling_with_scale<q8x16_t>(vres, quant_rescale, scale, new_offset);
                // Store result
                wrapper::vstore(reinterpret_cast<T *>(out.ptr()) + x_off, wrapper::vgetlow(requantized_dst));
                wrapper::vstore(reinterpret_cast<T *>(out.ptr()) + x_off + 8, wrapper::vgethigh(requantized_dst));
            }
            else
            {
                const float32x4_t scale_v = vdupq_n_f32(scale);
                // Divide by scale and add 0.5f to round to nearest instead of rounding towards zero
                vres1 = vcvtq_q32_f32<q32x4_t>(wrapper::vmla(half_scale_v, vcvtq_f32_q32(vres1), scale_v));
                vres2 = vcvtq_q32_f32<q32x4_t>(wrapper::vmla(half_scale_v, vcvtq_f32_q32(vres2), scale_v));
                vres3 = vcvtq_q32_f32<q32x4_t>(wrapper::vmla(half_scale_v, vcvtq_f32_q32(vres3), scale_v));
                vres4 = vcvtq_q32_f32<q32x4_t>(wrapper::vmla(half_scale_v, vcvtq_f32_q32(vres4), scale_v));

                const q8x8_t res1 = wrapper::vmovn(wrapper::vcombine(wrapper::vmovn(vres1), wrapper::vmovn(vres2)));
                const q8x8_t res2 = wrapper::vmovn(wrapper::vcombine(wrapper::vmovn(vres3), wrapper::vmovn(vres4)));
                // Store result
                wrapper::vstore(reinterpret_cast<T *>(out.ptr()) + x_off, res1);
                wrapper::vstore(reinterpret_cast<T *>(out.ptr()) + x_off + 8, res2);
            }
        }

        // Left-overs loop
        for(; x_off < window_end_x; ++x_off)
        {
            q32_t res = static_cast<q32_t>(0.f);

            // Perform pooling
            for(int z = pool_start_z; z < pool_end_z; ++z)
            {
                const uint8_t *in_ptr_z = in_ptr_n + (z + in_idx_depth) * w_stride;
                for(int y = pool_start_y; y < pool_end_y; ++y)
                {
                    const uint8_t *in_ptr_y = in_ptr_z + (y + in_idx_height) * z_stride;
                    for(int x = pool_start_x; x < pool_end_x; ++x)
                    {
                        const uint8_t *in_ptr_x = in_ptr_y + (x + in_idx_width) * y_stride;
                        const T        data     = *(reinterpret_cast<const T *>(in_ptr_x) + x_off);
                        res += data;
                    }
                }
            }

            if(src_qinfo != dst_qinfo)
            {
                const float res_f           = static_cast<float>(res);
                const float new_scale       = quant_rescale / scale;
                const auto  requantized_dst = quantize<T>(res_f, UniformQuantizationInfo(new_scale, new_offset));

                // Store result
                *(reinterpret_cast<T *>(out.ptr()) + x_off) = requantized_dst;
            }
            else
            {
                // Divide by scale and add 0.5f to round to nearest instead of rounding towards zero
                res = static_cast<T>(0.5f + static_cast<float>(res) * scale);

                // Store result
                *(reinterpret_cast<T *>(out.ptr()) + x_off) = res;
            }
        }
    },
    out);
}

bilinear_neon_scale()

void arm_compute::cpu::bilinear_neon_scale	(	const ITensor *	src,
		ITensor *	dst,
		const ITensor *	offsets,
		const ITensor *	dx,
		const ITensor *	dy,
		BorderMode	border_mode,
		PixelValue	constant_border_value,
		float	sampling_offset,
		bool	align_corners,
		const Window &	window
	)

Definition at line 143 of file list.h.

References ARM_COMPUTE_ERROR, ARM_COMPUTE_UNUSED, arm_compute::test::validation::b, arm_compute::scale_utils::calculate_resize_ratio(), arm_compute::CONSTANT, ITensorInfo::dimension(), Window::DimX, Window::DimY, Window::DimZ, Window::Dimension::end(), PixelValue::get(), ITensor::info(), Iterator::ptr(), arm_compute::REPLICATE, arm_compute::test::validation::scale_x, arm_compute::test::validation::scale_y, Window::set(), Window::Dimension::start(), Window::Dimension::step(), ITensorInfo::strides_in_bytes(), type, arm_compute::wrapper::vdup_n(), arm_compute::wrapper::vloadq(), arm_compute::wrapper::vmla(), arm_compute::wrapper::vstore(), Window::y(), and Window::z().

{
    ARM_COMPUTE_UNUSED(offsets);
    ARM_COMPUTE_UNUSED(dx);
    ARM_COMPUTE_UNUSED(dy);
    using ExactTagType = typename wrapper::traits::neon_bitvector_tag_t<T, wrapper::traits::BitWidth::W128>;

    // Compute the ratio between source and destination dimensions
    const float scale_x = scale_utils::calculate_resize_ratio(src->info()->dimension(1), dst->info()->dimension(1), align_corners);
    const float scale_y = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners);

    const int in_stride_y  = src->info()->strides_in_bytes()[1];
    const int in_stride_z  = src->info()->strides_in_bytes()[2];
    const int in_stride_w  = src->info()->strides_in_bytes()[3];
    const int out_stride_y = dst->info()->strides_in_bytes()[1];
    const int out_stride_z = dst->info()->strides_in_bytes()[2];
    const int out_stride_w = dst->info()->strides_in_bytes()[3];
    const int in_dim_w     = src->info()->dimension(1);
    const int in_dim_h     = src->info()->dimension(2);
    const int out_dim_ch   = dst->info()->dimension(0);
    const int step_cout    = 16 / sizeof(T);

    Window window_execution = window;
    window_execution.set(Window::DimX, Window::Dimension(0, 1, 1));
    Window win_in_out(window);
    win_in_out.set(Window::DimY, Window::Dimension(0, 0, 0));
    win_in_out.set(Window::DimZ, Window::Dimension(0, 0, 0));
    Iterator in(src, win_in_out);
    Iterator out(dst, win_in_out);

    const int xo_start = window_execution.y().start();
    const int xo_end   = window_execution.y().end();
    const int xo_step  = window_execution.y().step();
    const int yo_start = window_execution.z().start();
    const int yo_end   = window_execution.z().end();
    const int yo_step  = window_execution.z().step();
    const int bo_start = window_execution[3].start();
    const int bo_end   = window_execution[3].end();
    const int bo_step  = window_execution[3].step();

    if(border_mode == BorderMode::CONSTANT)
    {
#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
        using ConstType = typename std::conditional<std::is_same<T, float16_t>::value, half, T>::type;
#else  /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */
        using ConstType = T;
#endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */
        const T const_border_value = static_cast<T>(constant_border_value.get<ConstType>());

        for(int bo = bo_start; bo < bo_end; bo += bo_step)
        {
            const uint8_t *in_ptr_base  = in.ptr() + bo * in_stride_w;
            uint8_t       *out_ptr_base = out.ptr() + bo * out_stride_w;

            for(int yo = yo_start; yo < yo_end; yo += yo_step)
            {
                // Floating-point coordinate
                const float yi_f = ((yo + sampling_offset) * scale_y - sampling_offset);
                // Integer coordinate
                const auto yi = static_cast<int>(std::floor(yi_f));
                // Weight for the y coordinate
                const auto a1 = (yi_f - static_cast<float>(yi));
                const auto b1 = (1.f - a1);

                for(int xo = xo_start; xo < xo_end; xo += xo_step)
                {
                    // Floating-point coordinate
                    const float xi_f = ((xo + sampling_offset) * scale_x - sampling_offset);
                    // Integer coordinate
                    const auto xi = static_cast<int>(std::floor(xi_f));
                    // Weight for the x coordinate
                    const auto a = (xi_f - static_cast<float>(xi));
                    const auto b = (1.f - a);

                    const auto s00_s = static_cast<T>(b * b1);
                    const auto s01_s = static_cast<T>(a * b1);
                    const auto s10_s = static_cast<T>(b * a1);
                    const auto s11_s = static_cast<T>(a * a1);

                    const uint8_t *in_ptr  = in_ptr_base + xi * in_stride_y + yi * in_stride_z;
                    uint8_t       *out_ptr = out_ptr_base + xo * out_stride_y + yo * out_stride_z;

                    int cout = 0;
                    for(; cout <= (out_dim_ch - step_cout); cout += step_cout)
                    {
                        auto in00 = wrapper::vdup_n(static_cast<T>(const_border_value), ExactTagType{});
                        auto in01 = wrapper::vdup_n(static_cast<T>(const_border_value), ExactTagType{});
                        auto in10 = wrapper::vdup_n(static_cast<T>(const_border_value), ExactTagType{});
                        auto in11 = wrapper::vdup_n(static_cast<T>(const_border_value), ExactTagType{});
                        if((yi >= 0) && (yi < in_dim_h))
                        {
                            if((xi >= 0) && (xi < in_dim_w))
                            {
                                in00 = wrapper::vloadq(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T)));
                            }
                            if(((xi + 1) >= 0) && ((xi + 1) < in_dim_w))
                            {
                                in01 = wrapper::vloadq(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + in_stride_y));
                            }
                        }
                        if(((yi + 1) >= 0) && ((yi + 1) < in_dim_h))
                        {
                            if((xi >= 0) && (xi < in_dim_w))
                            {
                                in10 = wrapper::vloadq(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + in_stride_z));
                            }
                            if(((xi + 1) >= 0) && ((xi + 1) < in_dim_w))
                            {
                                in11 = wrapper::vloadq(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + in_stride_y + in_stride_z));
                            }
                        }

                        const auto s00  = wrapper::vdup_n(s00_s, ExactTagType{});
                        const auto s01  = wrapper::vdup_n(s01_s, ExactTagType{});
                        const auto s10  = wrapper::vdup_n(s10_s, ExactTagType{});
                        const auto s11  = wrapper::vdup_n(s11_s, ExactTagType{});
                        auto       out0 = wrapper::vdup_n(static_cast<T>(0), ExactTagType{});
                        out0            = wrapper::vmla(out0, in00, s00);
                        out0            = wrapper::vmla(out0, in01, s01);
                        out0            = wrapper::vmla(out0, in10, s10);
                        out0            = wrapper::vmla(out0, in11, s11);
                        wrapper::vstore(reinterpret_cast<T *>(out_ptr + cout * sizeof(T)), out0);
                    }

                    for(; cout < out_dim_ch; ++cout)
                    {
                        auto in00 = static_cast<T>(const_border_value);
                        auto in01 = static_cast<T>(const_border_value);
                        auto in10 = static_cast<T>(const_border_value);
                        auto in11 = static_cast<T>(const_border_value);
                        if((yi >= 0) && (yi < in_dim_h))
                        {
                            if((xi >= 0) && (xi < in_dim_w))
                            {
                                in00 = *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T)));
                            }
                            if(((xi + 1) >= 0) && ((xi + 1) < in_dim_w))
                            {
                                in01 = *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + in_stride_y));
                            }
                        }
                        if(((yi + 1) >= 0) && ((yi + 1) < in_dim_h))
                        {
                            if((xi >= 0) && (xi < in_dim_w))
                            {
                                in10 = *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + in_stride_z));
                            }
                            if(((xi + 1) >= 0) && ((xi + 1) < in_dim_w))
                            {
                                in11 = *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + in_stride_y + in_stride_z));
                            }
                        }
                        auto out0 = static_cast<T>(0);
                        out0 += in00 * s00_s;
                        out0 += in01 * s01_s;
                        out0 += in10 * s10_s;
                        out0 += in11 * s11_s;
                        *(reinterpret_cast<T *>(out_ptr + cout * sizeof(T))) = out0;
                    }
                }
            }
        }
    }
    else if(border_mode == BorderMode::REPLICATE)
    {
        for(int bo = bo_start; bo < bo_end; bo += bo_step)
        {
            const uint8_t *in_ptr  = in.ptr() + bo * in_stride_w;
            uint8_t       *out_ptr = out.ptr() + bo * out_stride_w;

            for(int yo = yo_start; yo < yo_end; yo += yo_step)
            {
                // Floating-point coordinate
                const float yi_f = ((yo + sampling_offset) * scale_y - sampling_offset);
                // Integer coordinate
                const auto yi = static_cast<int>(std::floor(yi_f));
                // Weight for the y coordinate
                const auto a1 = (yi_f - static_cast<float>(yi));
                const auto b1 = (1.f - a1);

                const auto yi0 = utility::clamp<int>(yi, 0, in_dim_h - 1);
                const auto yi1 = utility::clamp<int>(yi + 1, 0, in_dim_h - 1);

                for(int xo = xo_start; xo < xo_end; xo += xo_step)
                {
                    // Floating-point coordinate
                    const float xi_f = ((xo + sampling_offset) * scale_x - sampling_offset);
                    // Integer coordinate
                    const auto xi = static_cast<int>(std::floor(xi_f));
                    // Weight for the x coordinate
                    const auto a = (xi_f - static_cast<float>(xi));
                    const auto b = (1.f - a);

                    const auto s00_s = static_cast<T>(b * b1);
                    const auto s01_s = static_cast<T>(a * b1);
                    const auto s10_s = static_cast<T>(b * a1);
                    const auto s11_s = static_cast<T>(a * a1);

                    const auto xi0 = utility::clamp<int>(xi, 0, in_dim_w - 1);
                    const auto xi1 = utility::clamp<int>(xi + 1, 0, in_dim_w - 1);

                    int cout = 0;
                    for(; cout <= (out_dim_ch - step_cout); cout += step_cout)
                    {
                        auto in00 = wrapper::vdup_n(static_cast<T>(0), ExactTagType{});
                        auto in01 = wrapper::vdup_n(static_cast<T>(0), ExactTagType{});
                        auto in10 = wrapper::vdup_n(static_cast<T>(0), ExactTagType{});
                        auto in11 = wrapper::vdup_n(static_cast<T>(0), ExactTagType{});
                        in00      = wrapper::vloadq(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + (xi0) * in_stride_y + (yi0) * in_stride_z));
                        in01      = wrapper::vloadq(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + (xi1) * in_stride_y + (yi0) * in_stride_z));
                        in10      = wrapper::vloadq(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + (xi0) * in_stride_y + (yi1) * in_stride_z));
                        in11      = wrapper::vloadq(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + (xi1) * in_stride_y + (yi1) * in_stride_z));

                        const auto s00  = wrapper::vdup_n(s00_s, ExactTagType{});
                        const auto s01  = wrapper::vdup_n(s01_s, ExactTagType{});
                        const auto s10  = wrapper::vdup_n(s10_s, ExactTagType{});
                        const auto s11  = wrapper::vdup_n(s11_s, ExactTagType{});
                        auto       out0 = wrapper::vdup_n(static_cast<T>(0), ExactTagType{});
                        out0            = wrapper::vmla(out0, in00, s00);
                        out0            = wrapper::vmla(out0, in01, s01);
                        out0            = wrapper::vmla(out0, in10, s10);
                        out0            = wrapper::vmla(out0, in11, s11);
                        wrapper::vstore(reinterpret_cast<T *>(out_ptr + cout * sizeof(T) + xo * out_stride_y + yo * out_stride_z), out0);
                    }

                    for(; cout < out_dim_ch; ++cout)
                    {
                        auto in00 = static_cast<T>(0);
                        auto in01 = static_cast<T>(0);
                        auto in10 = static_cast<T>(0);
                        auto in11 = static_cast<T>(0);
                        in00      = *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + (xi0) * in_stride_y + (yi0) * in_stride_z));
                        in01      = *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + (xi1) * in_stride_y + (yi0) * in_stride_z));
                        in10      = *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + (xi0) * in_stride_y + (yi1) * in_stride_z));
                        in11      = *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + (xi1) * in_stride_y + (yi1) * in_stride_z));
                        auto out0 = static_cast<T>(0);
                        out0 += in00 * s00_s;
                        out0 += in01 * s01_s;
                        out0 += in10 * s10_s;
                        out0 += in11 * s11_s;
                        *(reinterpret_cast<T *>(out_ptr + cout * sizeof(T) + xo * out_stride_y + yo * out_stride_z)) = out0;
                    }
                }
            }
        }
    }
    else
    {
        ARM_COMPUTE_ERROR("Not implemented");
    }
}

bounding_box_transform()

void bounding_box_transform	(	const ITensor *	boxes,
		ITensor *	pred_boxes,
		const ITensor *	deltas,
		BoundingBoxTransformInfo	bbinfo,
		const Window &	window
	)

Definition at line 87 of file impl.cpp.

References BoundingBoxTransformInfo::apply_scale(), ARM_COMPUTE_ERROR_ON, BoundingBoxTransformInfo::bbox_xform_clip(), bounding_box_transform< float >(), ITensor::buffer(), BoundingBoxTransformInfo::correct_transform_coords(), arm_compute::execute_window_loop(), BoundingBoxTransformInfo::img_height(), BoundingBoxTransformInfo::img_width(), ITensor::info(), offset(), ITensorInfo::offset_first_element_in_bytes(), Iterator::ptr(), BoundingBoxTransformInfo::scale(), ITensorInfo::tensor_shape(), and BoundingBoxTransformInfo::weights().

{
    const size_t num_classes  = deltas->info()->tensor_shape()[0] >> 2;
    const size_t deltas_width = deltas->info()->tensor_shape()[0];
    const int    img_h        = std::floor(bbinfo.img_height() / bbinfo.scale() + 0.5f);
    const int    img_w        = std::floor(bbinfo.img_width() / bbinfo.scale() + 0.5f);

    const auto scale_after  = (bbinfo.apply_scale() ? T(bbinfo.scale()) : T(1));
    const auto scale_before = T(bbinfo.scale());
    ARM_COMPUTE_ERROR_ON(scale_before <= 0);
    const auto offset = (bbinfo.correct_transform_coords() ? T(1.f) : T(0.f));

    auto pred_ptr  = reinterpret_cast<T *>(pred_boxes->buffer() + pred_boxes->info()->offset_first_element_in_bytes());
    auto delta_ptr = reinterpret_cast<T *>(deltas->buffer() + deltas->info()->offset_first_element_in_bytes());

    Iterator box_it(boxes, window);
    execute_window_loop(window, [&](const Coordinates & id)
    {
        const auto ptr    = reinterpret_cast<T *>(box_it.ptr());
        const auto b0     = *ptr;
        const auto b1     = *(ptr + 1);
        const auto b2     = *(ptr + 2);
        const auto b3     = *(ptr + 3);
        const T    width  = (b2 / scale_before) - (b0 / scale_before) + T(1.f);
        const T    height = (b3 / scale_before) - (b1 / scale_before) + T(1.f);
        const T    ctr_x  = (b0 / scale_before) + T(0.5f) * width;
        const T    ctr_y  = (b1 / scale_before) + T(0.5f) * height;
        for(size_t j = 0; j < num_classes; ++j)
        {
            // Extract deltas
            const size_t delta_id = id.y() * deltas_width + 4u * j;
            const T      dx       = delta_ptr[delta_id] / T(bbinfo.weights()[0]);
            const T      dy       = delta_ptr[delta_id + 1] / T(bbinfo.weights()[1]);
            T            dw       = delta_ptr[delta_id + 2] / T(bbinfo.weights()[2]);
            T            dh       = delta_ptr[delta_id + 3] / T(bbinfo.weights()[3]);
            // Clip dw and dh
            dw = std::min(dw, T(bbinfo.bbox_xform_clip()));
            dh = std::min(dh, T(bbinfo.bbox_xform_clip()));
            // Determine the predictions
            const T pred_ctr_x = dx * width + ctr_x;
            const T pred_ctr_y = dy * height + ctr_y;
            const T pred_w     = std::exp(dw) * width;
            const T pred_h     = std::exp(dh) * height;
            // Store the prediction into the output tensor
            pred_ptr[delta_id]     = scale_after * utility::clamp<T>(pred_ctr_x - T(0.5f) * pred_w, T(0), T(img_w - 1));
            pred_ptr[delta_id + 1] = scale_after * utility::clamp<T>(pred_ctr_y - T(0.5f) * pred_h, T(0), T(img_h - 1));
            pred_ptr[delta_id + 2] = scale_after * utility::clamp<T>(pred_ctr_x + T(0.5f) * pred_w - offset, T(0), T(img_w - 1));
            pred_ptr[delta_id + 3] = scale_after * utility::clamp<T>(pred_ctr_y + T(0.5f) * pred_h - offset, T(0), T(img_h - 1));
        }
    },
    box_it);
}

bounding_box_transform< float >()

template void arm_compute::cpu::bounding_box_transform< float >	(	const ITensor *	boxes,
		ITensor *	pred_boxes,
		const ITensor *	deltas,
		BoundingBoxTransformInfo	bbinfo,
		const Window &	window
	)

Referenced by bounding_box_transform(), and neon_fp32_boundingboxtransform().

bounding_box_transform_qsymm16()

void bounding_box_transform_qsymm16	(	const ITensor *	boxes,
		ITensor *	pred_boxes,
		const ITensor *	deltas,
		BoundingBoxTransformInfo	bbinfo,
		const Window &	window
	)

Definition at line 29 of file impl.cpp.

References BoundingBoxTransformInfo::apply_scale(), BoundingBoxTransformInfo::bbox_xform_clip(), ITensor::buffer(), BoundingBoxTransformInfo::correct_transform_coords(), arm_compute::dequantize_qasymm16(), arm_compute::dequantize_qasymm8(), arm_compute::execute_window_loop(), BoundingBoxTransformInfo::img_height(), BoundingBoxTransformInfo::img_width(), ITensor::info(), offset(), ITensorInfo::offset_first_element_in_bytes(), Iterator::ptr(), ITensorInfo::quantization_info(), arm_compute::quantize_qasymm16(), BoundingBoxTransformInfo::scale(), ITensorInfo::tensor_shape(), QuantizationInfo::uniform(), and BoundingBoxTransformInfo::weights().

Referenced by neon_qu16_boundingboxtransform().

{
    const size_t num_classes  = deltas->info()->tensor_shape()[0] >> 2;
    const size_t deltas_width = deltas->info()->tensor_shape()[0];
    const int    img_h        = std::floor(bbinfo.img_height() / bbinfo.scale() + 0.5f);
    const int    img_w        = std::floor(bbinfo.img_width() / bbinfo.scale() + 0.5f);

    const auto scale_after  = (bbinfo.apply_scale() ? bbinfo.scale() : 1.f);
    const auto scale_before = bbinfo.scale();
    const auto offset       = (bbinfo.correct_transform_coords() ? 1.f : 0.f);

    auto pred_ptr  = reinterpret_cast<uint16_t *>(pred_boxes->buffer() + pred_boxes->info()->offset_first_element_in_bytes());
    auto delta_ptr = reinterpret_cast<uint8_t *>(deltas->buffer() + deltas->info()->offset_first_element_in_bytes());

    const auto boxes_qinfo  = boxes->info()->quantization_info().uniform();
    const auto deltas_qinfo = deltas->info()->quantization_info().uniform();
    const auto pred_qinfo   = pred_boxes->info()->quantization_info().uniform();

    Iterator box_it(boxes, window);
    execute_window_loop(window, [&](const Coordinates & id)
    {
        const auto  ptr    = reinterpret_cast<uint16_t *>(box_it.ptr());
        const auto  b0     = dequantize_qasymm16(*ptr, boxes_qinfo);
        const auto  b1     = dequantize_qasymm16(*(ptr + 1), boxes_qinfo);
        const auto  b2     = dequantize_qasymm16(*(ptr + 2), boxes_qinfo);
        const auto  b3     = dequantize_qasymm16(*(ptr + 3), boxes_qinfo);
        const float width  = (b2 / scale_before) - (b0 / scale_before) + 1.f;
        const float height = (b3 / scale_before) - (b1 / scale_before) + 1.f;
        const float ctr_x  = (b0 / scale_before) + 0.5f * width;
        const float ctr_y  = (b1 / scale_before) + 0.5f * height;
        for(size_t j = 0; j < num_classes; ++j)
        {
            // Extract deltas
            const size_t delta_id = id.y() * deltas_width + 4u * j;
            const float  dx       = dequantize_qasymm8(delta_ptr[delta_id], deltas_qinfo) / bbinfo.weights()[0];
            const float  dy       = dequantize_qasymm8(delta_ptr[delta_id + 1], deltas_qinfo) / bbinfo.weights()[1];
            float        dw       = dequantize_qasymm8(delta_ptr[delta_id + 2], deltas_qinfo) / bbinfo.weights()[2];
            float        dh       = dequantize_qasymm8(delta_ptr[delta_id + 3], deltas_qinfo) / bbinfo.weights()[3];
            // Clip dw and dh
            dw = std::min(dw, bbinfo.bbox_xform_clip());
            dh = std::min(dh, bbinfo.bbox_xform_clip());
            // Determine the predictions
            const float pred_ctr_x = dx * width + ctr_x;
            const float pred_ctr_y = dy * height + ctr_y;
            const float pred_w     = std::exp(dw) * width;
            const float pred_h     = std::exp(dh) * height;
            // Store the prediction into the output tensor
            pred_ptr[delta_id]     = quantize_qasymm16(scale_after * utility::clamp<float>(pred_ctr_x - 0.5f * pred_w, 0.f, img_w - 1.f), pred_qinfo);
            pred_ptr[delta_id + 1] = quantize_qasymm16(scale_after * utility::clamp<float>(pred_ctr_y - 0.5f * pred_h, 0.f, img_h - 1.f), pred_qinfo);
            pred_ptr[delta_id + 2] = quantize_qasymm16(scale_after * utility::clamp<float>(pred_ctr_x + 0.5f * pred_w - offset, 0.f, img_w - 1.f), pred_qinfo);
            pred_ptr[delta_id + 3] = quantize_qasymm16(scale_after * utility::clamp<float>(pred_ctr_y + 0.5f * pred_h - offset, 0.f, img_h - 1.f), pred_qinfo);
        }
    },
    box_it);
}

common_neon_scale()

void arm_compute::cpu::common_neon_scale	(	const ITensor *	src,
		ITensor *	dst,
		const ITensor *	offsets,
		const ITensor *	dx,
		const ITensor *	dy,
		InterpolationPolicy	policy,
		BorderMode	border_mode,
		PixelValue	constant_border_value,
		float	sampling_offset,
		bool	align_corners,
		const Window &	window
	)

Definition at line 398 of file list.h.

References arm_compute::BILINEAR, arm_compute::test::validation::dst, arm_compute::NEAREST_NEIGHBOR, and arm_compute::test::validation::src.

{
    if(policy == InterpolationPolicy::BILINEAR)
    {
        bilinear_neon_scale<T>(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window);
    }
    else if(policy == InterpolationPolicy::NEAREST_NEIGHBOR)
    {
        nearest_neon_scale<T>(src, dst, offsets, sampling_offset, align_corners, window);
    }
}

compute_all_anchors()

void compute_all_anchors	(	const ITensor *	anchors,
		ITensor *	all_anchors,
		ComputeAnchorsInfo	anchors_info,
		const Window &	window
	)

Definition at line 32 of file impl.cpp.

References compute_all_anchors< float >(), ITensorInfo::dimension(), arm_compute::execute_window_loop(), ComputeAnchorsInfo::feat_width(), ITensor::info(), Iterator::ptr(), ITensor::ptr_to_element(), ComputeAnchorsInfo::spatial_scale(), and Window::y().

{
    Iterator all_anchors_it(all_anchors, window);
    Iterator anchors_it(all_anchors, window);

    const size_t num_anchors = anchors->info()->dimension(1);
    const T      stride      = 1.f / anchors_info.spatial_scale();
    const size_t feat_width  = anchors_info.feat_width();

    execute_window_loop(window, [&](const Coordinates & id)
    {
        const size_t anchor_offset = id.y() % num_anchors;

        const auto out_anchor_ptr = reinterpret_cast<T *>(all_anchors_it.ptr());
        const auto anchor_ptr     = reinterpret_cast<T *>(anchors->ptr_to_element(Coordinates(0, anchor_offset)));

        const size_t shift_idy = id.y() / num_anchors;
        const T      shiftx    = (shift_idy % feat_width) * stride;
        const T      shifty    = (shift_idy / feat_width) * stride;

        *out_anchor_ptr       = *anchor_ptr + shiftx;
        *(out_anchor_ptr + 1) = *(1 + anchor_ptr) + shifty;
        *(out_anchor_ptr + 2) = *(2 + anchor_ptr) + shiftx;
        *(out_anchor_ptr + 3) = *(3 + anchor_ptr) + shifty;
    },
    all_anchors_it);
}

compute_all_anchors< float >()

template void arm_compute::cpu::compute_all_anchors< float >	(	const ITensor *	anchors,
		ITensor *	all_anchors,
		ComputeAnchorsInfo	anchors_info,
		const Window &	window
	)

Referenced by compute_all_anchors(), and neon_fp32_computeallanchors().

compute_all_anchors_qasymm16()

void compute_all_anchors_qasymm16	(	const ITensor *	anchors,
		ITensor *	all_anchors,
		ComputeAnchorsInfo	anchors_info,
		const Window &	window
	)

Definition at line 65 of file impl.cpp.

References arm_compute::dequantize_qsymm16(), ITensorInfo::dimension(), arm_compute::execute_window_loop(), ComputeAnchorsInfo::feat_width(), ITensor::info(), Iterator::ptr(), ITensor::ptr_to_element(), arm_compute::test::validation::qinfo, ITensorInfo::quantization_info(), arm_compute::quantize_qsymm16(), UniformQuantizationInfo::scale, ComputeAnchorsInfo::spatial_scale(), QuantizationInfo::uniform(), and Window::y().

Referenced by neon_qu16_computeallanchors().

{
    Iterator all_anchors_it(all_anchors, window);
    Iterator anchors_it(all_anchors, window);

    const size_t num_anchors = anchors->info()->dimension(1);
    const float  stride      = 1.f / anchors_info.spatial_scale();
    const size_t feat_width  = anchors_info.feat_width();

    const UniformQuantizationInfo qinfo = anchors->info()->quantization_info().uniform();

    execute_window_loop(window, [&](const Coordinates & id)
    {
        const size_t anchor_offset = id.y() % num_anchors;

        const auto out_anchor_ptr = reinterpret_cast<int16_t *>(all_anchors_it.ptr());
        const auto anchor_ptr     = reinterpret_cast<int16_t *>(anchors->ptr_to_element(Coordinates(0, anchor_offset)));

        const size_t shift_idy = id.y() / num_anchors;
        const float  shiftx    = (shift_idy % feat_width) * stride;
        const float  shifty    = (shift_idy / feat_width) * stride;

        const float new_anchor_x1 = dequantize_qsymm16(*anchor_ptr, qinfo.scale) + shiftx;
        const float new_anchor_y1 = dequantize_qsymm16(*(1 + anchor_ptr), qinfo.scale) + shifty;
        const float new_anchor_x2 = dequantize_qsymm16(*(2 + anchor_ptr), qinfo.scale) + shiftx;
        const float new_anchor_y2 = dequantize_qsymm16(*(3 + anchor_ptr), qinfo.scale) + shifty;

        *out_anchor_ptr       = quantize_qsymm16(new_anchor_x1, qinfo.scale);
        *(out_anchor_ptr + 1) = quantize_qsymm16(new_anchor_y1, qinfo.scale);
        *(out_anchor_ptr + 2) = quantize_qsymm16(new_anchor_x2, qinfo.scale);
        *(out_anchor_ptr + 3) = quantize_qsymm16(new_anchor_y2, qinfo.scale);
    },
    all_anchors_it);
}

compute_region_coordinate()

float arm_compute::cpu::compute_region_coordinate ( int  p, float  bin_size, float  roi_anchor, float  max_value  )

inline

Definition at line 206 of file impl.cpp.

References arm_compute::utility::clamp().

Referenced by roi_align().

{
    const float region_start = p * bin_size + roi_anchor;
    return utility::clamp(region_start, 0.0f, max_value);
}

directconv3d_float_neon_ndhwc()

void arm_compute::cpu::directconv3d_float_neon_ndhwc	(	const ITensor *	src0,
		const ITensor *	src1,
		const ITensor *	src2,
		ITensor *	dst,
		const Conv3dInfo &	conv_info,
		const Window &	window
	)

Definition at line 39 of file list.h.

References ITensor::buffer(), arm_compute::calculate_max_window(), conv_pad_left, conv_pad_top, Size3D::depth, ITensorInfo::dimension(), Window::DimW, Window::DimX, Window::DimY, Window::DimZ, ITensorInfo::element_size(), arm_compute::execute_window_loop(), Padding3D::front, Size3D::height, ITensor::info(), arm_compute::test::validation::k, Padding3D::left, ITensorInfo::offset_first_element_in_bytes(), Conv3dInfo::padding, Iterator::ptr(), Window::set(), arm_compute::test::validation::src, Conv3dInfo::stride, ITensorInfo::strides_in_bytes(), Padding3D::top, type, arm_compute::wrapper::vdup_n(), arm_compute::wrapper::vloadq(), arm_compute::wrapper::vmla(), arm_compute::vreduce(), arm_compute::wrapper::vsetlane(), Size3D::width, Dimensions< T >::y(), and Dimensions< T >::z().

{
    const ITensor *src     = src0;
    const ITensor *weights = src1;
    const ITensor *biases  = src2;

    using vtype                                = wrapper::traits::neon_bitvector<T, wrapper::traits::BitWidth::W128>;
    using vector_type                          = typename vtype::type;
    using tag_type                             = typename vtype::tag_type;
    constexpr int num_elems_read_per_iteration = 16 / sizeof(T);

    // Scalar quantities (N D H W Cin)
    const int element_size   = src->info()->element_size();
    const int input_stride_w = src->info()->strides_in_bytes().y() / element_size;
    const int input_stride_h = src->info()->strides_in_bytes().z() / element_size;
    const int input_stride_d = src->info()->strides_in_bytes()[3] / element_size;
    const int input_stride_n