51 AsmGemmInfo init_assembly_metadata(
const GEMMInfo &
info)
55 asm_info.reinterpret_input_as_3d = info.reinterpret_input_as_3d();
56 asm_info.depth_output_gemm3d = info.depth_output_gemm3d();
57 asm_info.activation_info = info.activation_info();
58 asm_info.output_stage = info.gemmlowp_output_stage();
69 : _memory_group(memory_manager), _weights_manager(weights_manager), _asm_glue(
std::make_unique<
NEGEMMAssemblyDispatch>(memory_manager, weights_manager)), _mm_kernel(), _mtx_a_reshape_kernel(),
70 _mtx_b_reshape_kernel(), _mtx_a_reduction_kernel(), _mtx_b_reduction_kernel(), _offset_contribution_kernel(), _offset_contribution_output_stage_kernel(), _activation_func(),
71 _convert_to_signed_asymm(), _convert_from_signed_asymm(), _vector_sum_col(), _vector_sum_row(), _tmp_a(), _tmp_b(), _mm_result_s32(), _signed_a(), _signed_output(), _original_b(nullptr), _a_offset(0),
72 _b_offset(0), _run_vector_matrix_multiplication(false), _assembly_path(false), _fused_assembly_path(false), _reshape_b_only_on_first_run(false), _is_prepared(false), _fuse_output_stage(false),
73 _run_activation(false), _flip_signedness(false)
90 _run_vector_matrix_multiplication = a->
info()->
dimension(1) < 2;
93 _fused_assembly_path =
false;
102 const int32_t offset_correction = 128;
107 _memory_group.
manage(&_signed_a);
108 _convert_to_signed_asymm = std::make_unique<NEConvertQuantizedSignednessKernel>();
109 _convert_to_signed_asymm->configure(a_to_use, &_signed_a);
110 a_to_use = &_signed_a;
114 _memory_group.
manage(&_signed_output);
125 matrix_a = &_signed_a;
131 _fuse_output_stage =
true;
132 _memory_group.
manage(&_mm_result_s32);
138 const AsmGemmInfo asm_info = init_assembly_metadata(gemm_info);
149 _asm_glue->configure(a_to_use, b, c, output, asm_info);
150 _fused_assembly_path = _asm_glue->is_configured();
154 _asm_glue->configure(a_to_use, b,
nullptr, _fuse_output_stage ? &_mm_result_s32 : output, asm_info);
156 _assembly_path = _asm_glue->is_configured();
166 if(!(_assembly_path || _run_vector_matrix_multiplication))
177 _memory_group.
manage(&_tmp_a);
178 if(!_reshape_b_only_on_first_run)
180 _memory_group.
manage(&_tmp_b);
184 _mtx_a_reshape_kernel = std::make_unique<NEGEMMInterleave4x4Kernel>();
185 _mtx_a_reshape_kernel->configure(a_to_use, &_tmp_a);
188 _mtx_b_reshape_kernel = std::make_unique<NEGEMMTranspose1xWKernel>();
189 _mtx_b_reshape_kernel->configure(b, &_tmp_b);
192 if(!_fused_assembly_path)
203 if(!_reshape_b_only_on_first_run)
205 _memory_group.
manage(&_vector_sum_col);
209 _mtx_b_reduction_kernel = std::make_unique<NEGEMMLowpMatrixBReductionKernel>();
210 _mtx_b_reduction_kernel->configure(b, &_vector_sum_col, reduction_info);
219 _memory_group.
manage(&_vector_sum_row);
222 _mtx_a_reduction_kernel = std::make_unique<NEGEMMLowpMatrixAReductionKernel>();
223 _mtx_a_reduction_kernel->configure(a_to_use, &_vector_sum_row, reduction_info);
226 if(_fuse_output_stage)
231 _mm_kernel = std::make_unique<NEGEMMLowpMatrixMultiplyKernel>();
232 _mm_kernel->configure(matrix_a, matrix_b, &_mm_result_s32);
235 _offset_contribution_output_stage_kernel = std::make_unique<NEGEMMLowpOffsetContributionOutputStageKernel>();
236 _offset_contribution_output_stage_kernel->configure(&_mm_result_s32,
237 _a_offset == 0 ?
nullptr : &_vector_sum_col,
238 _b_offset == 0 ?
nullptr : &_vector_sum_row, c,
239 _flip_signedness ? &_signed_output : output,
245 _convert_from_signed_asymm = std::make_unique<NEConvertQuantizedSignednessKernel>();
246 _convert_from_signed_asymm->configure(&_signed_output, output);
254 _mm_kernel = std::make_unique<NEGEMMLowpMatrixMultiplyKernel>();
255 _mm_kernel->configure(matrix_a, matrix_b, output);
258 _offset_contribution_kernel = std::make_unique<NEGEMMLowpOffsetContributionKernel>();
259 _offset_contribution_kernel->configure(output, _a_offset == 0 ?
nullptr : &_vector_sum_col, _b_offset == 0 ?
nullptr : &_vector_sum_row, a_to_use->
info()->
dimension(0), _a_offset, _b_offset);
267 _activation_func.
configure(output,
nullptr, activation);
271 if(!_assembly_path && !_run_vector_matrix_multiplication)
274 if(!_reshape_b_only_on_first_run)
280 if(!_fused_assembly_path)
282 if(_a_offset != 0 && !_reshape_b_only_on_first_run)
293 if(_fuse_output_stage)
312 "The product AB is defined only if the number of columns in A is equal to the number of rows in B");
330 if(fuse_output_stage)
341 const int32_t offset_correction = 128;
347 a_to_use = &signed_a;
355 output_stage_corr.
gemmlowp_offset = signed_output.quantization_info().uniform().offset;
358 info.set_gemmlowp_output_stage(output_stage_corr);
361 matrix_a_info = &signed_a;
365 const AsmGemmInfo asm_info = init_assembly_metadata(info);
368 bool run_optimised =
false;
369 bool run_optimised_requantized =
false;
373 run_optimised_requantized = run_optimised;
383 if(info.depth_output_gemm3d() != 0)
385 if(info.reinterpret_input_as_3d())
405 const bool run_vector_matrix_multiplication = a->
dimension(1) < 2;
406 if(!run_vector_matrix_multiplication)
408 matrix_a_info = &tmp_a_info;
409 matrix_b_info = &tmp_b_info;
430 if(!run_optimised_requantized)
455 if(fuse_output_stage)
467 a_offset == 0 ?
nullptr : &info_vector_sum_col,
468 b_offset == 0 ?
nullptr : &info_vector_sum_row,
470 flip_signedness ? &signed_output : output,
472 info.gemmlowp_output_stage()));
485 a_offset == 0 ?
nullptr : &info_vector_sum_col,
486 b_offset == 0 ?
nullptr : &info_vector_sum_row,
487 a_offset, b_offset));
514 if(_asm_glue->is_configured())
520 if(!_run_vector_matrix_multiplication)
525 if(!_reshape_b_only_on_first_run)
534 if(!_fused_assembly_path)
543 if(_a_offset != 0 && !_reshape_b_only_on_first_run)
548 if(_fuse_output_stage)
561 if(!_fused_assembly_path && _fuse_output_stage && _flip_signedness)
569 _activation_func.
run();
577 const bool original_b_managed_by_weights_manager = _weights_manager && _weights_manager->
are_weights_managed(_original_b);
579 if(_asm_glue->is_configured())
581 if(!original_b_managed_by_weights_manager)
586 _asm_glue->prepare();
587 if(!original_b_managed_by_weights_manager)
593 else if(_reshape_b_only_on_first_run && !_run_vector_matrix_multiplication && !_asm_glue->is_configured())
595 if(!original_b_managed_by_weights_manager)
603 if(!original_b_managed_by_weights_manager)
610 if(!_fused_assembly_path && _a_offset != 0 && _reshape_b_only_on_first_run)
void prepare() override
Prepare the function for executing.
Quantize using a fixed point multiplication.
void init(const TensorAllocator &allocator, const Coordinates &coords, TensorInfo &sub_info)
Shares the same backing memory with another tensor allocator, while the tensor info might be differen...
bool enabled() const
Check if initialised.
virtual size_t dimension(size_t index) const =0
Return the size of the requested dimension.
#define ARM_COMPUTE_ERROR(msg)
Print the given message then throw an std::runtime_error.
1 channel, 1 U8 per channel
#define ARM_COMPUTE_RETURN_ON_ERROR(status)
Checks if a status contains an error and returns it.
virtual DataType data_type() const =0
Data type used for each element of the tensor.
bool is_used() const
Flags if the tensor is used or not.
NEGEMMLowpMatrixMultiplyCore(std::shared_ptr< IMemoryManager > memory_manager=nullptr, IWeightsManager *weights_manager=nullptr)
Constructor.
GEMMLowpOutputStageInfo gemmlowp_output_stage() const
GEMMLowp output stage.
TensorShape compute_reductionA_shape(const ITensorInfo &b)
Calculate the reductionA shape used in GEMMLowp.
static Status validate(const ITensorInfo *input, const ITensorInfo *output, const ActivationLayerInfo &act_info)
[NEActivationLayer snippet]
#define ARM_COMPUTE_ERROR_ON(cond)
If the condition is true then an error message is printed and an exception thrown.
Store the tensor's metadata.
#define ARM_COMPUTE_ERROR_THROW_ON(status)
int32_t gemmlowp_offset
GEMMLowp output stage offset used for quantizing to QASYMM8.
int32_t gemmlowp_max_bound
GEMMLowp max value used to saturate down the output result before converting back to QASYMM8...
static Status validate(const ITensorInfo *mtx_b, const ITensorInfo *vector_sum_col, const GEMMLowpReductionKernelInfo &info)
Static function to check if given info will lead to a valid configuration of NEGEMMLowpMatrixBReducti...
#define ARM_COMPUTE_RETURN_ERROR_ON(cond)
If the condition is true, an error is returned.
Activation Layer Information class.
GEMMLowpOutputStageType type
GEMMLowp output stage type.
Interface for Neon tensor.
TensorShape compute_interleaved_shape(const ITensorInfo &a, int mult_interleave4x4_height=1, bool reinterpret_input_as_3d=false)
Calculate the interleaved shape of an input tensor.
static Status validate(const ITensorInfo *mm_result, const ITensorInfo *vector_sum_col, const ITensorInfo *vector_sum_row, int32_t a_offset, int32_t b_offset)
Static function to check if given info will lead to a valid configuration of NEGEMMLowpOffsetContribu...
static Status validate(const ITensorInfo *input, const ITensorInfo *output)
Static function to check if given info will lead to a valid configuration of NEGEMMTranspose1xWKernel...
Copyright (c) 2017-2021 Arm Limited.
bool is_b_reshaped() const
Flag which specifies if the matrix B has been reshaped.
TensorAllocator * allocator()
Return a pointer to the tensor's allocator.
ITensorInfo * info() const override
Interface to be implemented by the child class to return the tensor's metadata.
void mark_as_unused() const
Marks a tensor as unused.
1 channel, 1 S32 per channel
void manage(IMemoryManageable *obj) override
Sets a object to be managed by the given memory group.
bool are_weights_managed(const ITensor *weights)
Check if the weights are managed.
static Status validate(const ITensorInfo *input, const ITensorInfo *output)
Static function to check if given info will lead to a valid configuration of NEConvertQuantizedSigned...
Quantization information.
static constexpr size_t DimX
Alias for dimension 0 also known as X dimension.
#define ARM_COMPUTE_UNUSED(...)
To avoid unused variables warnings.
static bool is_activation_supported(const ActivationLayerInfo &activation)
Checks if activation is supported by the gemm assembly dispatcher.
void configure(const ITensor *a, const ITensor *b, const ITensor *c, ITensor *output, const GEMMInfo &gemm_info=GEMMInfo())
Initialise the kernel's inputs, output.
bool is_data_type_quantized_per_channel(DataType dt)
Check if a given data type is of per channel type.
void run() override
Run the kernels contained in the function.
virtual const TensorShape & tensor_shape() const =0
Size for each dimension of the tensor.
void run() override
Run the kernels contained in the function.
quantized, asymmetric fixed-point 8-bit number unsigned
static Status validate(const ITensorInfo *input, const ITensorInfo *output)
Static function to check if given info will lead to a valid configuration of NEGEMMInterleave4x4Kerne...
void allocate() override
Allocate size specified by TensorInfo of CPU memory.
UniformQuantizationInfo uniform() const
Return per layer quantization info.
bool auto_init_if_empty(ITensorInfo &info, const TensorShape &shape, int num_channels, DataType data_type, QuantizationInfo quantization_info=QuantizationInfo())
Auto initialize the tensor info (shape, number of channels and data type) if the current assignment i...
virtual std::unique_ptr< T > clone() const =0
Provide a clone of the current object of class T.
GEMMLowp output stage info.
virtual ITensorInfo * info() const =0
Interface to be implemented by the child class to return the tensor's metadata.
static Status validate(const ITensorInfo *a, const ITensorInfo *b, const ITensorInfo *c, const ITensorInfo *d, const AsmGemmInfo &info)
Indicates whether or not this function can be used to process the given parameters.
Weights manager interface to handle weights transformations.
virtual QuantizationInfo quantization_info() const =0
Get the quantization settings (scale and offset) of the tensor.
quantized, symmetric fixed-point 8-bit number
bool is_data_type_quantized_asymmetric(DataType dt)
Check if a given data type is of asymmetric quantized type.
bool is_a_reshaped() const
Flag which specifies if the matrix A has been reshaped.
quantized, symmetric per channel fixed-point 8-bit number
TensorShape compute_reductionB_shape(const ITensorInfo &a)
Calculate the reductionB shape used in GEMMLowp.
static constexpr size_t DimY
Alias for dimension 1 also known as Y dimension.
ScaleKernelInfo info(interpolation_policy, default_border_mode, PixelValue(), sampling_policy, false)
Memory group resources scope handling class.
void set_gemmlowp_output_stage(GEMMLowpOutputStageInfo &output_stage)
Sets GEMMLowp output stage.
virtual void schedule(ICPPKernel *kernel, const Hints &hints)=0
Runs the kernel in the same thread as the caller synchronously.
#define ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(t, c,...)
void configure(ITensor *input, ITensor *output, ActivationLayerInfo activation_info)
[NEActivationLayer snippet]
TensorShape compute_transpose1xW_shape(const ITensorInfo &b)
Calculate the transposed 1xW shape.
#define ARM_COMPUTE_RETURN_ERROR_ON_MSG(cond, msg)
If the condition is true, an error is returned.
#define ARM_COMPUTE_ERROR_ON_NULLPTR(...)
Store the tensor's metadata.
bool reshape_b_only_on_first_run() const
Flag which specifies if the reshape of matrix B should executed only for the first.
static Status validate(const ITensorInfo *a, const ITensorInfo *b, const ITensorInfo *c, const ITensorInfo *output, const GEMMInfo &gemm_info=GEMMInfo())
Static function to check if given info will lead to a valid configuration of NEGEMMLowpMatrixMultiply...
static Status validate(const ITensorInfo *input0, const ITensorInfo *input1, const ITensorInfo *output)
Static function to check if given info will lead to a valid configuration of NEGEMMLowpMatrixMultiply...
quantized, asymmetric fixed-point 8-bit number signed
static Status validate(const ITensorInfo *mm_result, const ITensorInfo *vector_sum_col, const ITensorInfo *vector_sum_row, const ITensorInfo *bias, const ITensorInfo *output, int32_t a_offset, int32_t b_offset, GEMMLowpOutputStageInfo output_stage)
Static function to check if given info will lead to a valid configuration of NEGEMMLowpOffsetContribu...
static Status validate(const ITensorInfo *mtx_a, const ITensorInfo *vector_sum_row, const GEMMLowpReductionKernelInfo &info)
Static function to check if given info will lead to a valid configuration of NEGEMMLowpMatrixAReducti...
int32_t gemmlowp_min_bound
GEMMLowp min value used to saturate down the output result before converting back to QASYMM8...
DataType
Available data types.
ActivationLayerInfo activation_info() const
Activation layer to apply after the matrix multiplication.
~NEGEMMLowpMatrixMultiplyCore()
Default destructor.
TensorShape & set(size_t dimension, size_t value, bool apply_dim_correction=true, bool increase_dim_unit=true)
Accessor to set the value of one of the dimensions.
static IScheduler & get()
Access the scheduler singleton.