33 #include "src/core/NEON/kernels/assembly/NEDepthwiseConvolutionAssemblyKernelWrapper.h" 34 #include "src/core/NEON/kernels/convolution/depthwise/depthwise_dilated.hpp" 35 #include "src/core/NEON/kernels/convolution/depthwise/depthwise_quantized_dilated.hpp" 48 std::unique_ptr<depthwise::IDepthwiseConvolution> get_qasymm8_convolver(
int kernel_size,
int stride_x,
49 int n_batches,
int in_rows,
int in_cols,
int n_channels,
50 int dilation_factor, neon_convolution_kernels::ActivationFunction activation,
51 const qasymm8::QAsymm8Params &wqinfo,
const qasymm8::QAsymm8Params &iqinfo,
const qasymm8::QAsymm8Params &oqinfo,
52 const qasymm8::QAsymm8RescaleParams &rescale_params,
53 int padding_top,
int padding_left,
int padding_bottom,
int padding_right)
62 return std::make_unique<depthwise::QAsymm8DilatedDepthwiseConvolution<2, 2, 3, 3, 1, 1>>(
63 n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, wqinfo, iqinfo, oqinfo, rescale_params, padding_top, padding_left, padding_bottom, padding_right);
65 return std::make_unique<depthwise::QAsymm8DilatedDepthwiseConvolution<2, 2, 3, 3, 2, 2>>(
66 n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, wqinfo, iqinfo, oqinfo, rescale_params, padding_top, padding_left, padding_bottom, padding_right);
76 return std::make_unique<depthwise::QAsymm8DilatedDepthwiseConvolution<2, 2, 5, 5, 1, 1>>(
77 n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, wqinfo, iqinfo, oqinfo, rescale_params, padding_top, padding_left, padding_bottom, padding_right);
79 return std::make_unique<depthwise::QAsymm8DilatedDepthwiseConvolution<2, 2, 5, 5, 2, 2>>(
80 n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, wqinfo, iqinfo, oqinfo, rescale_params, padding_top, padding_left, padding_bottom, padding_right);
90 std::unique_ptr<depthwise::IDepthwiseConvolution> get_qsymm8_perchannel_convolver(
int kernel_size,
int stride_x,
91 int n_batches,
int in_rows,
int in_cols,
int n_channels,
92 neon_convolution_kernels::ActivationFunction activation,
93 const qsymm8::QSymm8PerChannelParams &wqinfo,
const qasymm8::QAsymm8Params &iqinfo,
const qasymm8::QAsymm8Params &oqinfo,
94 const qsymm8::QSymm8PerChannelRescaleParams &rescale_params,
95 int padding_top,
int padding_left,
int padding_bottom,
int padding_right)
104 return std::make_unique<depthwise::QSymm8HybridPerChannelDepthwiseConvolution<2, 2, 3, 3, 1, 1>>(
105 n_batches, in_rows, in_cols, n_channels, activation, wqinfo, iqinfo, oqinfo, rescale_params, padding_top, padding_left, padding_bottom, padding_right);
107 return std::make_unique<depthwise::QSymm8HybridPerChannelDepthwiseConvolution<2, 2, 3, 3, 2, 2>>(
108 n_batches, in_rows, in_cols, n_channels, activation, wqinfo, iqinfo, oqinfo, rescale_params, padding_top, padding_left, padding_bottom, padding_right);
118 return std::make_unique<depthwise::QSymm8HybridPerChannelDepthwiseConvolution<2, 2, 5, 5, 1, 1>>(
119 n_batches, in_rows, in_cols, n_channels, activation, wqinfo, iqinfo, oqinfo, rescale_params, padding_top, padding_left, padding_bottom, padding_right);
121 return std::make_unique<depthwise::QSymm8HybridPerChannelDepthwiseConvolution<2, 2, 5, 5, 2, 2>>(
122 n_batches, in_rows, in_cols, n_channels, activation, wqinfo, iqinfo, oqinfo, rescale_params, padding_top, padding_left, padding_bottom, padding_right);
132 #ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC 133 std::unique_ptr<depthwise::IDepthwiseConvolution> get_fp16_convolver(
int kernel_size,
int stride_x,
134 int n_batches,
int in_rows,
int in_cols,
int n_channels,
135 int dilation_factor, neon_convolution_kernels::ActivationFunction activation,
136 int padding_top,
int padding_left,
int padding_bottom,
int padding_right)
145 return std::make_unique<depthwise::DilatedDepthwiseConvolution<3, 3, 3, 3, 1, 1, float16_t, float16_t, float16_t>>(
146 n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, padding_top, padding_left, padding_bottom, padding_right);
148 return std::make_unique<depthwise::DilatedDepthwiseConvolution<3, 3, 3, 3, 2, 2, float16_t, float16_t, float16_t>>(
149 n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, padding_top, padding_left, padding_bottom, padding_right);
159 return std::make_unique<depthwise::DilatedDepthwiseConvolution<3, 3, 5, 5, 1, 1, float16_t, float16_t, float16_t>>(
160 n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, padding_top, padding_left, padding_bottom, padding_right);
162 return std::make_unique<depthwise::DilatedDepthwiseConvolution<3, 3, 5, 5, 2, 2, float16_t, float16_t, float16_t>>(
163 n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, padding_top, padding_left, padding_bottom, padding_right);
172 #endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC 174 std::unique_ptr<depthwise::IDepthwiseConvolution> get_fp32_convolver(
int kernel_size,
int stride_x,
175 int n_batches,
int in_rows,
int in_cols,
int n_channels,
176 int dilation_factor, neon_convolution_kernels::ActivationFunction activation,
177 int padding_top,
int padding_left,
int padding_bottom,
int padding_right)
186 return std::make_unique<depthwise::DilatedDepthwiseConvolution<4, 4, 3, 3, 1, 1, float, float, float>>(
187 n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, padding_top, padding_left, padding_bottom, padding_right);
189 return std::make_unique<depthwise::DilatedDepthwiseConvolution<3, 3, 3, 3, 2, 2, float, float, float>>(
190 n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, padding_top, padding_left, padding_bottom, padding_right);
200 return std::make_unique<depthwise::DilatedDepthwiseConvolution<4, 4, 5, 5, 1, 1, float, float, float>>(
201 n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, padding_top, padding_left, padding_bottom, padding_right);
203 return std::make_unique<depthwise::DilatedDepthwiseConvolution<3, 3, 5, 5, 2, 2, float, float, float>>(
204 n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, padding_top, padding_left, padding_bottom, padding_right);
214 std::unique_ptr<depthwise::IDepthwiseConvolution> create_convolver(
const ITensorInfo *
input,
215 const ITensorInfo *weights,
217 const ConvolutionInfo &
info)
220 const TensorShape
shape =
input->tensor_shape();
222 const int n_batches =
shape[3];
223 const int in_rows =
shape.z();
224 const int in_cols =
shape.y();
225 const int n_channels =
shape.x();
226 const int dilation_factor =
info.dilation.x();
227 const int padding_top =
info.pad_stride_info.pad_top();
228 const int padding_left =
info.pad_stride_info.pad_left();
229 const int padding_bottom =
info.pad_stride_info.pad_bottom();
230 const int padding_right =
info.pad_stride_info.pad_right();
235 const unsigned int stride_x =
info.pad_stride_info.stride().first;
236 const unsigned int kernel_size = weights->tensor_shape().y();
242 activation = neon_convolution_kernels::ActivationFunction::ReLU;
246 activation = neon_convolution_kernels::ActivationFunction::ReLU6;
250 if(is_uniform_quantized)
252 const UniformQuantizationInfo input_qinfo =
input->quantization_info().uniform();
253 const UniformQuantizationInfo weights_qinfo = weights->quantization_info().uniform();
254 const UniformQuantizationInfo output_qinfo = output->quantization_info().uniform();
260 const qasymm8::QAsymm8Params iqinfo{ static_cast<uint8_t>(input_qinfo.offset), input_qinfo.scale };
261 const qasymm8::QAsymm8Params wqinfo{ static_cast<uint8_t>(weights_qinfo.offset), weights_qinfo.scale };
262 const qasymm8::QAsymm8Params oqinfo{ static_cast<uint8_t>(output_qinfo.offset), output_qinfo.scale };
265 const float fmultipler = iqinfo.scale * wqinfo.scale / oqinfo.scale;
266 int32_t qmultiplier = 0;
269 qasymm8::QAsymm8RescaleParams rescale_params(qshift, qmultiplier, fmultipler);
271 return get_qasymm8_convolver(kernel_size, stride_x, n_batches, in_rows, in_cols, n_channels, dilation_factor, activation,
272 wqinfo, iqinfo, oqinfo, rescale_params, padding_top, padding_left, padding_bottom, padding_right);
274 else if(is_perchannel_quantized)
276 const UniformQuantizationInfo input_qinfo =
input->quantization_info().uniform();
277 const QuantizationInfo weights_qinfo = weights->quantization_info();
278 const UniformQuantizationInfo output_qinfo = output->quantization_info().uniform();
283 const qasymm8::QAsymm8Params iqinfo{ static_cast<uint8_t>(input_qinfo.offset), input_qinfo.scale };
284 const qsymm8::QSymm8PerChannelParams wqinfo{ weights_qinfo.scale() };
285 const qasymm8::QAsymm8Params oqinfo{ static_cast<uint8_t>(output_qinfo.offset), output_qinfo.scale };
288 std::vector<float> fmultipliers;
289 std::vector<int32_t> qmultipliers;
290 std::vector<int32_t> qshifts;
292 for(
auto const s : wqinfo.scales)
294 const float fmultipler = iqinfo.scale * s / oqinfo.scale;
295 int32_t qmultiplier = 0;
298 fmultipliers.push_back(fmultipler);
299 qmultipliers.push_back(qmultiplier);
300 qshifts.push_back(qshift);
303 qsymm8::QSymm8PerChannelRescaleParams rescale_params(qshifts, qmultipliers, fmultipliers);
305 return get_qsymm8_perchannel_convolver(kernel_size, stride_x, n_batches, in_rows, in_cols, n_channels, activation,
306 wqinfo, iqinfo, oqinfo, rescale_params, padding_top, padding_left, padding_bottom, padding_right);
313 #ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC 316 return get_fp16_convolver(kernel_size, stride_x, n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, padding_top, padding_left, padding_bottom, padding_right);
318 #endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC 321 return get_fp32_convolver(kernel_size, stride_x, n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, padding_top, padding_left, padding_bottom, padding_right);
330 struct CpuDepthwiseConvolutionAssemblyDispatch::LocalImpl
332 std::unique_ptr<depthwise::IDepthwiseConvolution> dwc_assembly_kernel{
nullptr };
333 NEDepthwiseConvolutionAssemblyKernelWrapper dwc_acl_kernel{};
334 bool is_prepared{
false };
338 #ifndef DOXYGEN_SKIP_THIS 340 : _pImpl(std::make_unique<LocalImpl>())
357 bias !=
nullptr ? bias :
nullptr,
365 _pImpl->is_prepared =
false;
368 _pImpl->dwc_assembly_kernel = create_convolver(
input, weights, output,
info);
372 _pImpl->dwc_acl_kernel.configure(_pImpl->dwc_assembly_kernel.get());
374 constexpr
size_t alignment = 128;
378 const size_t workspace_size = _pImpl->dwc_assembly_kernel->get_working_space_size(num_threads);
383 const size_t pack_tensor_size = _pImpl->dwc_assembly_kernel->get_packed_params_size();
391 return _pImpl->mem_req;
436 for(
auto const s : weights_qinfo.
scale())
438 const float fmultipler = input_qinfo.
scale * s / output_qinfo.
scale;
469 std::set<unsigned int> supported_kernel_sizes = { 3, 5 };
472 const unsigned int kernel_w = weights->
dimension(width_idx);
473 const unsigned int kernel_h = weights->
dimension(height_idx);
474 bool weights_supported = (kernel_w == kernel_h) && (supported_kernel_sizes.count(kernel_w) != 0);
477 const auto &strides =
info.pad_stride_info.stride();
478 bool supported_strides = (strides.first == strides.second) && ((strides.first == 1) || (strides.first == 2));
481 const auto pad_top =
info.pad_stride_info.pad_top();
482 const auto pad_right =
info.pad_stride_info.pad_right();
483 const auto pad_bottom =
info.pad_stride_info.pad_bottom();
484 const auto pad_left =
info.pad_stride_info.pad_left();
486 bool is_same_padding = (pad_top == same_pad.
pad_top()) && (pad_right == same_pad.
pad_right()) && (pad_bottom == same_pad.
pad_bottom()) && (pad_left == same_pad.
pad_left());
487 bool is_valid_padding = (pad_top == 0) && (pad_right == 0) && (pad_bottom == 0) && (pad_left == 0);
488 bool supported_padding = is_same_padding || is_valid_padding;
490 bool is_dilation_supported = ((
info.dilation ==
Size2D(1
U, 1
U)) || ((
info.dilation.x() ==
info.dilation.y()) && strides.first == 1));
494 is_dilation_supported = is_dilation_supported && (
info.dilation ==
Size2D(1
U, 1
U));
497 return is_input_type_valid && is_weights_type_valid && weights_supported && supported_strides && supported_padding && (
info.depth_multiplier == 1) && is_dilation_supported;
511 _pImpl->dwc_assembly_kernel->set_working_space(static_cast<void *>(
workspace->buffer()));
514 const int input_element_size =
src->info()->element_size();
515 const int input_batch_stride =
src->info()->strides_in_bytes()[3] / input_element_size;
516 const int input_row_stride =
src->info()->strides_in_bytes().z() / input_element_size;
517 const int input_col_stride =
src->info()->strides_in_bytes().y() / input_element_size;
518 const void *input_ptr =
src->buffer() +
src->info()->offset_first_element_in_bytes();
519 _pImpl->dwc_assembly_kernel->set_input(input_ptr, input_batch_stride, input_row_stride, input_col_stride);
522 const int output_element_size =
dst->info()->element_size();
523 const int output_batch_stride =
dst->info()->strides_in_bytes()[3] / output_element_size;
524 const int output_row_stride =
dst->info()->strides_in_bytes().z() / output_element_size;
525 const int output_col_stride =
dst->info()->strides_in_bytes().y() / output_element_size;
526 void *output_ptr =
dst->buffer() +
dst->info()->offset_first_element_in_bytes();
527 _pImpl->dwc_assembly_kernel->set_output(output_ptr, output_batch_stride, output_row_stride, output_col_stride);
535 if(!_pImpl->is_prepared)
544 const int weights_element_size = weights->info()->element_size();
545 const int weights_row_stride = weights->info()->strides_in_bytes().z() / weights_element_size;
546 const int weights_col_stride = weights->info()->strides_in_bytes().y() / weights_element_size;
547 _pImpl->dwc_assembly_kernel->pack_params(packed_weights->buffer(),
548 weights->buffer() + weights->info()->offset_first_element_in_bytes(),
551 (bias !=
nullptr) ? bias->buffer() :
nullptr);
552 _pImpl->dwc_assembly_kernel->set_packed_params_buffer(packed_weights->buffer());
554 weights->mark_as_unused();
557 bias->mark_as_unused();
559 _pImpl->is_prepared =
true;
static Status validate(const ITensorInfo *input, const ITensorInfo *weights, const ITensorInfo *bias, const ITensorInfo *output, const ConvolutionInfo &info)
Static function to check if given info will lead to a valid configuration of CpuDepthwiseConvolutionA...
virtual size_t num_dimensions() const =0
The number of dimensions of the tensor (rank)
TensorShape compute_depthwise_convolution_shape(const ITensorInfo &input, const ITensorInfo &weights, const ConvolutionInfo &info)
Calculate the depthwise convolution output shape of a tensor.
void run(ITensorPack &tensors) override
Run the kernels contained in the function.
experimental::MemoryRequirements workspace() const override
Return the memory requirements required by the workspace.
#define ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_LAYOUT(...)
#define ARM_COMPUTE_RETURN_ERROR_ON_CPU_F16_UNSUPPORTED(tensor)
virtual size_t dimension(size_t index) const =0
Return the size of the requested dimension.
void prepare(ITensorPack &tensors) override
Prepare the function for executing.
~CpuDepthwiseConvolutionAssemblyDispatch()
Default destructor.
virtual DataType data_type() const =0
Data type used for each element of the tensor.
1 channel, 1 F32 per channel
#define ARM_COMPUTE_ERROR_ON(cond)
If the condition is true then an error message is printed and an exception thrown.
const DataLayout data_layout
Store the tensor's metadata.
#define ARM_COMPUTE_ERROR_THROW_ON(status)
unsigned int pad_top() const
Get the top padding.
static bool is_optimized_supported(const ITensorInfo *input, const ITensorInfo *weights, const ConvolutionInfo &info)
Check if the optimized kernel can be used for the given kernel sizes and strides.
#define ARM_COMPUTE_RETURN_ERROR_ON(cond)
If the condition is true, an error is returned.
#define ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DIMENSIONS(...)
SimpleTensor< float > src
Copyright (c) 2017-2021 Arm Limited.
std::vector< MemoryInfo > MemoryRequirements
1 channel, 1 F16 per channel
const ITensor * get_const_tensor(int id) const
Get constant tensor of a given id.
Quantization information.
static constexpr size_t DimX
Alias for dimension 0 also known as X dimension.
#define ARM_COMPUTE_UNUSED(...)
To avoid unused variables warnings.
virtual const TensorShape & tensor_shape() const =0
Size for each dimension of the tensor.
quantized, asymmetric fixed-point 8-bit number unsigned
#define ARM_COMPUTE_ERROR_ON_MSG(cond, msg)
UniformQuantizationInfo uniform() const
Return per layer quantization info.
bool auto_init_if_empty(ITensorInfo &info, const TensorShape &shape, int num_channels, DataType data_type, QuantizationInfo quantization_info=QuantizationInfo())
Auto initialize the tensor info (shape, number of channels and data type) if the current assignment i...
unsigned int pad_right() const
Get the right padding.
const std::vector< float > & scale() const
Scale vector accessor.
Padding and stride information class.
virtual QuantizationInfo quantization_info() const =0
Get the quantization settings (scale and offset) of the tensor.
Num samples, channels, height, width.
bool is_relu6(ActivationLayerInfo activation_info)
Checks if activation information correspond to a relu6 activation function.
quantized, symmetric per channel fixed-point 8-bit number
static constexpr size_t DimY
Alias for dimension 1 also known as Y dimension.
ScaleKernelInfo info(interpolation_policy, default_border_mode, PixelValue(), sampling_policy, false)
ITensor * get_tensor(int id)
Get tensor of a given id from the pac.
virtual size_t total_size() const =0
Returns the total size of the tensor in bytes.
virtual void schedule(ICPPKernel *kernel, const Hints &hints)=0
Runs the kernel in the same thread as the caller synchronously.
static constexpr size_t DimZ
Alias for dimension 2 also known as Z dimension.
Class for specifying the size of an image or rectangle.
#define ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(...)
Num samples, height, width, channels.
#define ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(t, c,...)
void configure(const ITensorInfo *input, const ITensorInfo *weights, const ITensorInfo *bias, ITensorInfo *output, const ConvolutionInfo &info)
Initialize the function's source, destination, kernels and border_size.
bool is_relu(ActivationLayerInfo activation_info)
Checks if activation information correspond to a relu activation function.
#define ARM_COMPUTE_ERROR_ON_NULLPTR(...)
quantized, asymmetric fixed-point 8-bit number signed
virtual unsigned int num_threads() const =0
Returns the number of threads that the SingleThreadScheduler has in his pool.
size_t get_data_layout_dimension_index(const DataLayout data_layout, const DataLayoutDimension data_layout_dimension)
Get the index of the given dimension.
Status calculate_quantized_multiplier_less_than_one(float multiplier, int32_t *quant_multiplier, int32_t *right_shift, bool ignore_epsilon=false)
Calculate quantized representation of multiplier with value less than one.
CpuDepthwiseConvolutionAssemblyDispatch()
unsigned int pad_bottom() const
Get the bottom padding.
DataType
Available data types.
unsigned int pad_left() const
Get the left padding.
DataLayout
[DataLayout enum definition]
bool is_data_type_float(DataType dt)
Check if a given data type is of floating point type.
PadStrideInfo calculate_same_pad(TensorShape input_shape, TensorShape weights_shape, PadStrideInfo conv_info, DataLayout data_layout=DataLayout::NCHW, const Size2D &dilation=Size2D(1u, 1u), const DimensionRoundingType &rounding_type=DimensionRoundingType::FLOOR)
Calculate padding requirements in case of SAME padding.
static IScheduler & get()
Access the scheduler singleton.