33 #include "src/core/NEON/kernels/assembly/NEDepthwiseConvolutionAssemblyKernelWrapper.h" 34 #include "src/core/NEON/kernels/convolution/depthwise/depthwise_dilated.hpp" 35 #include "src/core/NEON/kernels/convolution/depthwise/depthwise_quantized_dilated.hpp" 46 std::unique_ptr<depthwise::IDepthwiseConvolution> get_qasymm8_convolver(
int kernel_size,
int stride_x,
47 int n_batches,
int in_rows,
int in_cols,
int n_channels,
48 int dilation_factor, neon_convolution_kernels::ActivationFunction activation,
49 const qasymm8::QAsymm8Params &wqinfo,
const qasymm8::QAsymm8Params &iqinfo,
const qasymm8::QAsymm8Params &oqinfo,
50 const qasymm8::QAsymm8RescaleParams &rescale_params,
51 int padding_top,
int padding_left,
int padding_bottom,
int padding_right)
60 return std::make_unique<depthwise::QAsymm8DilatedDepthwiseConvolution<2, 2, 3, 3, 1, 1>>(
61 n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, wqinfo, iqinfo, oqinfo, rescale_params, padding_top, padding_left, padding_bottom, padding_right);
63 return std::make_unique<depthwise::QAsymm8DilatedDepthwiseConvolution<2, 2, 3, 3, 2, 2>>(
64 n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, wqinfo, iqinfo, oqinfo, rescale_params, padding_top, padding_left, padding_bottom, padding_right);
74 return std::make_unique<depthwise::QAsymm8DilatedDepthwiseConvolution<2, 2, 5, 5, 1, 1>>(
75 n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, wqinfo, iqinfo, oqinfo, rescale_params, padding_top, padding_left, padding_bottom, padding_right);
77 return std::make_unique<depthwise::QAsymm8DilatedDepthwiseConvolution<2, 2, 5, 5, 2, 2>>(
78 n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, wqinfo, iqinfo, oqinfo, rescale_params, padding_top, padding_left, padding_bottom, padding_right);
88 std::unique_ptr<depthwise::IDepthwiseConvolution> get_qsymm8_perchannel_convolver(
int kernel_size,
int stride_x,
89 int n_batches,
int in_rows,
int in_cols,
int n_channels,
90 neon_convolution_kernels::ActivationFunction activation,
91 const qsymm8::QSymm8PerChannelParams &wqinfo,
const qasymm8::QAsymm8Params &iqinfo,
const qasymm8::QAsymm8Params &oqinfo,
92 const qsymm8::QSymm8PerChannelRescaleParams &rescale_params,
93 int padding_top,
int padding_left,
int padding_bottom,
int padding_right)
102 return std::make_unique<depthwise::QSymm8HybridPerChannelDepthwiseConvolution<2, 2, 3, 3, 1, 1>>(
103 n_batches, in_rows, in_cols, n_channels, activation, wqinfo, iqinfo, oqinfo, rescale_params, padding_top, padding_left, padding_bottom, padding_right);
105 return std::make_unique<depthwise::QSymm8HybridPerChannelDepthwiseConvolution<2, 2, 3, 3, 2, 2>>(
106 n_batches, in_rows, in_cols, n_channels, activation, wqinfo, iqinfo, oqinfo, rescale_params, padding_top, padding_left, padding_bottom, padding_right);
116 return std::make_unique<depthwise::QSymm8HybridPerChannelDepthwiseConvolution<2, 2, 5, 5, 1, 1>>(
117 n_batches, in_rows, in_cols, n_channels, activation, wqinfo, iqinfo, oqinfo, rescale_params, padding_top, padding_left, padding_bottom, padding_right);
119 return std::make_unique<depthwise::QSymm8HybridPerChannelDepthwiseConvolution<2, 2, 5, 5, 2, 2>>(
120 n_batches, in_rows, in_cols, n_channels, activation, wqinfo, iqinfo, oqinfo, rescale_params, padding_top, padding_left, padding_bottom, padding_right);
130 #ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC 131 std::unique_ptr<depthwise::IDepthwiseConvolution> get_fp16_convolver(
int kernel_size,
int stride_x,
132 int n_batches,
int in_rows,
int in_cols,
int n_channels,
133 int dilation_factor, neon_convolution_kernels::ActivationFunction activation,
134 int padding_top,
int padding_left,
int padding_bottom,
int padding_right)
143 return std::make_unique<depthwise::DilatedDepthwiseConvolution<3, 3, 3, 3, 1, 1, float16_t, float16_t, float16_t>>(
144 n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, padding_top, padding_left, padding_bottom, padding_right);
146 return std::make_unique<depthwise::DilatedDepthwiseConvolution<3, 3, 3, 3, 2, 2, float16_t, float16_t, float16_t>>(
147 n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, padding_top, padding_left, padding_bottom, padding_right);
157 return std::make_unique<depthwise::DilatedDepthwiseConvolution<3, 3, 5, 5, 1, 1, float16_t, float16_t, float16_t>>(
158 n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, padding_top, padding_left, padding_bottom, padding_right);
160 return std::make_unique<depthwise::DilatedDepthwiseConvolution<3, 3, 5, 5, 2, 2, float16_t, float16_t, float16_t>>(
161 n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, padding_top, padding_left, padding_bottom, padding_right);
170 #endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC 172 std::unique_ptr<depthwise::IDepthwiseConvolution> get_fp32_convolver(
int kernel_size,
int stride_x,
173 int n_batches,
int in_rows,
int in_cols,
int n_channels,
174 int dilation_factor, neon_convolution_kernels::ActivationFunction activation,
175 int padding_top,
int padding_left,
int padding_bottom,
int padding_right)
184 return std::make_unique<depthwise::DilatedDepthwiseConvolution<4, 4, 3, 3, 1, 1, float, float, float>>(
185 n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, padding_top, padding_left, padding_bottom, padding_right);
187 return std::make_unique<depthwise::DilatedDepthwiseConvolution<3, 3, 3, 3, 2, 2, float, float, float>>(
188 n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, padding_top, padding_left, padding_bottom, padding_right);
198 return std::make_unique<depthwise::DilatedDepthwiseConvolution<4, 4, 5, 5, 1, 1, float, float, float>>(
199 n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, padding_top, padding_left, padding_bottom, padding_right);
201 return std::make_unique<depthwise::DilatedDepthwiseConvolution<3, 3, 5, 5, 2, 2, float, float, float>>(
202 n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, padding_top, padding_left, padding_bottom, padding_right);
212 std::unique_ptr<depthwise::IDepthwiseConvolution> create_convolver(
const ITensor *
input,
213 const ITensor *weights,
216 ActivationLayerInfo act_info,
217 const Size2D &dilation)
221 const TensorShape
shape = input->info()->tensor_shape();
223 const int n_batches = shape[3];
224 const int in_rows = shape.z();
225 const int in_cols = shape.y();
226 const int n_channels = shape.x();
227 const int dilation_factor = dilation.x();
228 const int padding_top = conv_info.pad_top();
229 const int padding_left = conv_info.pad_left();
230 const int padding_bottom = conv_info.pad_bottom();
231 const int padding_right = conv_info.pad_right();
236 const unsigned int stride_x = conv_info.stride().first;
237 const unsigned int kernel_size = weights->info()->tensor_shape().y();
243 activation = neon_convolution_kernels::ActivationFunction::ReLU;
247 activation = neon_convolution_kernels::ActivationFunction::ReLU6;
251 if(is_uniform_quantized)
253 const UniformQuantizationInfo input_qinfo = input->info()->quantization_info().uniform();
254 const UniformQuantizationInfo weights_qinfo = weights->info()->quantization_info().uniform();
255 const UniformQuantizationInfo output_qinfo = output->info()->quantization_info().uniform();
261 const qasymm8::QAsymm8Params iqinfo{
static_cast<uint8_t
>(input_qinfo.offset), input_qinfo.scale };
262 const qasymm8::QAsymm8Params wqinfo{
static_cast<uint8_t
>(weights_qinfo.offset), weights_qinfo.scale };
263 const qasymm8::QAsymm8Params oqinfo{
static_cast<uint8_t
>(output_qinfo.offset), output_qinfo.scale };
266 const float fmultipler = iqinfo.scale * wqinfo.scale / oqinfo.scale;
267 int32_t qmultiplier = 0;
270 qasymm8::QAsymm8RescaleParams rescale_params(qshift, qmultiplier, fmultipler);
272 return get_qasymm8_convolver(kernel_size, stride_x, n_batches, in_rows, in_cols, n_channels, dilation_factor, activation,
273 wqinfo, iqinfo, oqinfo, rescale_params, padding_top, padding_left, padding_bottom, padding_right);
275 else if(is_perchannel_quantized)
277 const UniformQuantizationInfo input_qinfo = input->info()->quantization_info().uniform();
278 const QuantizationInfo weights_qinfo = weights->info()->quantization_info();
279 const UniformQuantizationInfo output_qinfo = output->info()->quantization_info().uniform();
284 const qasymm8::QAsymm8Params iqinfo{
static_cast<uint8_t
>(input_qinfo.offset), input_qinfo.scale };
285 const qsymm8::QSymm8PerChannelParams wqinfo{ weights_qinfo.scale() };
286 const qasymm8::QAsymm8Params oqinfo{
static_cast<uint8_t
>(output_qinfo.offset), output_qinfo.scale };
289 std::vector<float> fmultipliers;
290 std::vector<int32_t> qmultipliers;
291 std::vector<int32_t> qshifts;
293 for(
auto const s : wqinfo.scales)
295 const float fmultipler = iqinfo.scale * s / oqinfo.scale;
296 int32_t qmultiplier = 0;
299 fmultipliers.push_back(fmultipler);
300 qmultipliers.push_back(qmultiplier);
301 qshifts.push_back(qshift);
304 qsymm8::QSymm8PerChannelRescaleParams rescale_params(qshifts, qmultipliers, fmultipliers);
306 return get_qsymm8_perchannel_convolver(kernel_size, stride_x, n_batches, in_rows, in_cols, n_channels, activation,
307 wqinfo, iqinfo, oqinfo, rescale_params, padding_top, padding_left, padding_bottom, padding_right);
314 #ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC 317 return get_fp16_convolver(kernel_size, stride_x, n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, padding_top, padding_left, padding_bottom, padding_right);
319 #endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC 322 return get_fp32_convolver(kernel_size, stride_x, n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, padding_top, padding_left, padding_bottom, padding_right);
331 struct NEDepthwiseConvolutionAssemblyDispatch::LocalImpl
333 std::unique_ptr<depthwise::IDepthwiseConvolution> _dwc_assembly_kernel{
nullptr };
334 NEDepthwiseConvolutionAssemblyKernelWrapper _dwc_acl_kernel{};
337 #ifndef DOXYGEN_SKIP_THIS 339 : _memory_group(
std::move(memory_manager)), _input(nullptr), _weights(nullptr), _bias(nullptr), _output(nullptr), _packed_weights(), _workspace(), _is_prepared(false),
340 _pImpl(
std::make_unique<LocalImpl>())
352 unsigned int depth_multiplier,
360 bias !=
nullptr ? bias->
info() :
nullptr,
375 _is_prepared =
false;
378 _pImpl->_dwc_assembly_kernel = create_convolver(input, weights, output, conv_info, act_info, dilation);
382 _pImpl->_dwc_acl_kernel.configure(_pImpl->_dwc_assembly_kernel.get());
384 constexpr
size_t alignment = 128;
388 const size_t workspace_size = _pImpl->_dwc_assembly_kernel->get_working_space_size(num_threads);
391 _memory_group.manage(&_workspace);
392 _workspace.allocator()->allocate();
395 const size_t pack_tensor_size = _pImpl->_dwc_assembly_kernel->get_packed_params_size();
405 unsigned int depth_multiplier,
445 for(
auto const s : weights_qinfo.
scale())
447 const float fmultipler = input_qinfo.
scale * s / output_qinfo.
scale;
457 unsigned int depth_multiplier,
481 std::set<unsigned int> supported_kernel_sizes = { 3, 5 };
484 const unsigned int kernel_w = weights->
dimension(width_idx);
485 const unsigned int kernel_h = weights->
dimension(height_idx);
486 bool weights_supported = (kernel_w == kernel_h) && (supported_kernel_sizes.count(kernel_w) != 0);
489 const auto &strides = conv_info.
stride();
490 bool supported_strides = (strides.first == strides.second) && ((strides.first == 1) || (strides.first == 2));
493 const auto pad_top = conv_info.
pad_top();
494 const auto pad_right = conv_info.
pad_right();
495 const auto pad_bottom = conv_info.
pad_bottom();
496 const auto pad_left = conv_info.
pad_left();
498 bool is_same_padding = (pad_top == same_pad.
pad_top()) && (pad_right == same_pad.
pad_right()) && (pad_bottom == same_pad.
pad_bottom()) && (pad_left == same_pad.
pad_left());
499 bool is_valid_padding = (pad_top == 0) && (pad_right == 0) && (pad_bottom == 0) && (pad_left == 0);
500 bool supported_padding = is_same_padding || is_valid_padding;
502 bool is_dilation_supported = ((dilation ==
Size2D(1
U, 1
U)) || ((dilation.
x() == dilation.
y()) && strides.first == 1));
506 is_dilation_supported = is_dilation_supported && (dilation ==
Size2D(1
U, 1
U));
509 return is_input_type_valid && is_weights_type_valid && weights_supported && supported_strides && supported_padding && (depth_multiplier == 1) && is_dilation_supported;
521 _pImpl->_dwc_assembly_kernel->set_working_space(static_cast<void *>(_workspace.buffer()));
524 const int input_element_size = _input->info()->element_size();
525 const int input_batch_stride = _input->info()->strides_in_bytes()[3] / input_element_size;
526 const int input_row_stride = _input->info()->strides_in_bytes().z() / input_element_size;
527 const int input_col_stride = _input->info()->strides_in_bytes().y() / input_element_size;
528 const void *input_ptr = _input->buffer() + _input->info()->offset_first_element_in_bytes();
529 _pImpl->_dwc_assembly_kernel->set_input(input_ptr, input_batch_stride, input_row_stride, input_col_stride);
532 const int output_element_size = _output->info()->element_size();
533 const int output_batch_stride = _output->info()->strides_in_bytes()[3] / output_element_size;
534 const int output_row_stride = _output->info()->strides_in_bytes().z() / output_element_size;
535 const int output_col_stride = _output->info()->strides_in_bytes().y() / output_element_size;
536 void *output_ptr = _output->buffer() + _output->info()->offset_first_element_in_bytes();
537 _pImpl->_dwc_assembly_kernel->set_output(output_ptr, output_batch_stride, output_row_stride, output_col_stride);
547 _packed_weights.allocator()->allocate();
551 const int weights_element_size = _weights->info()->element_size();
552 const int weights_row_stride = _weights->info()->strides_in_bytes().z() / weights_element_size;
553 const int weights_col_stride = _weights->info()->strides_in_bytes().y() / weights_element_size;
554 _pImpl->_dwc_assembly_kernel->pack_params(_packed_weights.buffer(),
555 _weights->buffer() + _weights->info()->offset_first_element_in_bytes(),
558 (_bias !=
nullptr) ? _bias->buffer() :
nullptr);
559 _pImpl->_dwc_assembly_kernel->set_packed_params_buffer(_packed_weights.buffer());
561 _weights->mark_as_unused();
564 _bias->mark_as_unused();
virtual size_t num_dimensions() const =0
The number of dimensions of the tensor (rank)
#define ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_LAYOUT(...)
#define ARM_COMPUTE_RETURN_ERROR_ON_CPU_F16_UNSUPPORTED(tensor)
TensorShape compute_depthwise_convolution_shape(const ITensorInfo &input, const ITensorInfo &weights, PadStrideInfo conv_info, unsigned int depth_multiplier, const Size2D &dilation=Size2D(1U, 1U))
Calculate the depthwise convolution output shape of a tensor.
bool enabled() const
Check if initialised.
virtual size_t dimension(size_t index) const =0
Return the size of the requested dimension.
virtual DataType data_type() const =0
Data type used for each element of the tensor.
static Status validate(const ITensorInfo *input, const ITensorInfo *weights, const ITensorInfo *bias, const ITensorInfo *output, const PadStrideInfo &conv_info, unsigned int depth_multiplier=1, const ActivationLayerInfo &act_info=ActivationLayerInfo(), const Size2D &dilation=Size2D(1, 1))
Static function to check if given info will lead to a valid configuration of NEDepthwiseConvolutionAs...
1 channel, 1 F32 per channel
#define ARM_COMPUTE_ERROR_ON(cond)
If the condition is true then an error message is printed and an exception thrown.
void prepare() override
Prepare the function for executing.
const DataLayout data_layout
Store the tensor's metadata.
#define ARM_COMPUTE_ERROR_THROW_ON(status)
size_t x() const
Semantic accessor for width as x.
unsigned int pad_top() const
Get the top padding.
#define ARM_COMPUTE_RETURN_ERROR_ON(cond)
If the condition is true, an error is returned.
Activation Layer Information class.
Interface for Neon tensor.
#define ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DIMENSIONS(...)
Copyright (c) 2017-2021 Arm Limited.
1 channel, 1 F16 per channel
NEDepthwiseConvolutionAssemblyDispatch(std::shared_ptr< IMemoryManager > memory_manager=nullptr)
Default constructor.
T x() const
Alias to access the size of the first dimension.
Quantization information.
static constexpr size_t DimX
Alias for dimension 0 also known as X dimension.
#define ARM_COMPUTE_UNUSED(...)
To avoid unused variables warnings.
virtual const TensorShape & tensor_shape() const =0
Size for each dimension of the tensor.
quantized, asymmetric fixed-point 8-bit number unsigned
T z() const
Alias to access the size of the third dimension.
#define ARM_COMPUTE_ERROR_ON_MSG(cond, msg)
std::pair< unsigned int, unsigned int > stride() const
Get the stride.
UniformQuantizationInfo uniform() const
Return per layer quantization info.
bool auto_init_if_empty(ITensorInfo &info, const TensorShape &shape, int num_channels, DataType data_type, QuantizationInfo quantization_info=QuantizationInfo())
Auto initialize the tensor info (shape, number of channels and data type) if the current assignment i...
virtual std::unique_ptr< T > clone() const =0
Provide a clone of the current object of class T.
virtual ITensorInfo * info() const =0
Interface to be implemented by the child class to return the tensor's metadata.
unsigned int pad_right() const
Get the right padding.
const std::vector< float > & scale() const
Scale vector accessor.
void configure(const ITensor *input, const ITensor *weights, const ITensor *bias, ITensor *output, const PadStrideInfo &conv_info, unsigned int depth_multiplier=1, const ActivationLayerInfo &act_info=ActivationLayerInfo(), const Size2D &dilation=Size2D(1, 1))
Initialize the function's source, destination, kernels and border_size.
Padding and stride information class.
static bool is_optimized_supported(const ITensorInfo *input, const ITensorInfo *weights, PadStrideInfo conv_info, unsigned int depth_multiplier=1, const Size2D &dilation=Size2D(1, 1))
Check if the optimized kernel can be used for the given kernel sizes and strides. ...
virtual QuantizationInfo quantization_info() const =0
Get the quantization settings (scale and offset) of the tensor.
Num samples, channels, height, width.
size_t y() const
Semantic accessor for height as y.
bool is_relu6(ActivationLayerInfo activation_info)
Checks if activation information correspond to a relu6 activation function.
quantized, symmetric per channel fixed-point 8-bit number
static constexpr size_t DimY
Alias for dimension 1 also known as Y dimension.
Memory group resources scope handling class.
virtual size_t total_size() const =0
Returns the total size of the tensor in bytes.
virtual void schedule(ICPPKernel *kernel, const Hints &hints)=0
Runs the kernel in the same thread as the caller synchronously.
static constexpr size_t DimZ
Alias for dimension 2 also known as Z dimension.
Class for specifying the size of an image or rectangle.
#define ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(...)
Num samples, height, width, channels.
#define ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(t, c,...)
bool is_relu(ActivationLayerInfo activation_info)
Checks if activation information correspond to a relu activation function.
#define ARM_COMPUTE_ERROR_ON_NULLPTR(...)
Store the tensor's metadata.
T y() const
Alias to access the size of the second dimension.
quantized, asymmetric fixed-point 8-bit number signed
virtual unsigned int num_threads() const =0
Returns the number of threads that the SingleThreadScheduler has in his pool.
size_t get_data_layout_dimension_index(const DataLayout data_layout, const DataLayoutDimension data_layout_dimension)
Get the index of the given dimension.
Status calculate_quantized_multiplier_less_than_one(float multiplier, int32_t *quant_multiplier, int32_t *right_shift, bool ignore_epsilon=false)
Calculate quantized representation of multiplier with value less than one.
void run() override
Run the kernels contained in the function.
unsigned int pad_bottom() const
Get the bottom padding.
DataType
Available data types.
unsigned int pad_left() const
Get the left padding.
DataLayout
[DataLayout enum definition]
TensorShape & set(size_t dimension, size_t value, bool apply_dim_correction=true, bool increase_dim_unit=true)
Accessor to set the value of one of the dimensions.
bool is_data_type_float(DataType dt)
Check if a given data type is of floating point type.
~NEDepthwiseConvolutionAssemblyDispatch()
Default destructor.
PadStrideInfo calculate_same_pad(TensorShape input_shape, TensorShape weights_shape, PadStrideInfo conv_info, DataLayout data_layout=DataLayout::NCHW, const Size2D &dilation=Size2D(1u, 1u), const DimensionRoundingType &rounding_type=DimensionRoundingType::FLOOR)
Calculate padding requirements in case of SAME padding.
virtual DataLayout data_layout() const =0
Get the data layout of the tensor.
static IScheduler & get()
Access the scheduler singleton.