44 Status
validate_arguments(
const ITensorInfo *mm_result,
const ITensorInfo *vector_sum_col,
const ITensorInfo *vector_sum_row,
45 int32_t a_offset, int32_t b_offset)
62 const bool reinterpret_as_3d = mm_result->num_dimensions() > 1 && mm_result->tensor_shape().y() != vector_sum_row->tensor_shape().x();
65 ARM_COMPUTE_RETURN_ERROR_ON(reinterpret_as_3d && vector_sum_row->dimension(0) != (mm_result->dimension(1) * mm_result->dimension(2)));
69 if(output_shape.num_dimensions() > 1)
71 const unsigned int output_batch_idx = reinterpret_as_3d ? 3 : 2;
73 TensorShape vector_sum_row_shape = vector_sum_row->tensor_shape();
75 output_shape.collapse_from(output_batch_idx);
78 "mm_result tensor must have the same number of batches of output tensor");
82 TensorShape vector_sum_col_shape = vector_sum_col->tensor_shape();
83 vector_sum_col_shape.collapse_from(1);
86 "vector_sum_col tensor must have the same number of batches of vector_sum_row_shape or the number of batches must be set to 1");
94 void run_offset_contribution(
const Window &window,
95 ITensor *mm_result,
const ITensor *vector_sum_col,
const ITensor *vector_sum_row,
96 int32_t a_offset, int32_t b_offset, int32_t k_offset,
bool slide_vector_sum_col,
bool is_gemm3d)
98 Window collapsed_window = window.collapse_if_possible(window,
Window::DimZ);
99 collapsed_window.set(
Window::DimX, Window::Dimension(0, 1, 1));
101 const int height_input = is_gemm3d ? mm_result->info()->dimension(1) : 0;
102 const int depth_input = is_gemm3d ? mm_result->info()->dimension(2) : 1;
104 const int window_start_x = window.x().start();
105 const int window_end_x = window.x().end();
106 const int window_step_x = 16;
108 Iterator mm_result_it(mm_result, collapsed_window);
110 if((a_offset != 0) && (b_offset != 0) && (vector_sum_col !=
nullptr) && (vector_sum_row !=
nullptr))
113 Window win_vector_sum_col(collapsed_window);
114 win_vector_sum_col.set(
Window::DimY, Window::Dimension(0, 0, 0));
115 win_vector_sum_col.set(
Window::DimZ, Window::Dimension(0, 0, 0));
118 Window win_vector_sum_row(collapsed_window);
119 win_vector_sum_row.set(
Window::DimX, Window::Dimension(0, 0, 0));
120 win_vector_sum_row.set(
Window::DimY, Window::Dimension(0, 0, 0));
121 win_vector_sum_row.set(
Window::DimZ, Window::Dimension(0, 0, 0));
123 Iterator vector_sum_col_it(vector_sum_col, win_vector_sum_col);
124 Iterator vector_sum_row_it(vector_sum_row, win_vector_sum_row);
126 const size_t sum_row_stride_y = vector_sum_row->info()->strides_in_bytes().y();
129 const int vector_sum_col_batch_offset = slide_vector_sum_col ? vector_sum_col->info()->strides_in_bytes().z() : 0;
133 const int batch_id =
id.z() / depth_input;
134 auto vector_sum_col_ptr =
reinterpret_cast<const int32_t *
>(vector_sum_col_it.ptr() + batch_id * vector_sum_col_batch_offset);
135 auto mm_result_ptr =
reinterpret_cast<int32_t *
>(mm_result_it.ptr());
138 int32_t b_offset_term_s32 = *(
reinterpret_cast<const int32_t *
>(vector_sum_row_it.ptr() + batch_id * sum_row_stride_y) +
id.y() + (
id.z() % depth_input) * height_input);
139 b_offset_term_s32 *= b_offset;
141 const int32x4_t b_offset_term_s32_vec = vdupq_n_s32(b_offset_term_s32);
143 int x = window_start_x;
144 for(; x <= (window_end_x - window_step_x); x += window_step_x)
147 int32x4x4_t a_offset_term_s32 =
150 vld1q_s32(vector_sum_col_ptr + x + 0),
151 vld1q_s32(vector_sum_col_ptr + x + 4),
152 vld1q_s32(vector_sum_col_ptr + x + 8),
153 vld1q_s32(vector_sum_col_ptr + x + 12)
157 a_offset_term_s32.val[0] = vmulq_n_s32(a_offset_term_s32.val[0], a_offset);
158 a_offset_term_s32.val[1] = vmulq_n_s32(a_offset_term_s32.val[1], a_offset);
159 a_offset_term_s32.val[2] = vmulq_n_s32(a_offset_term_s32.val[2], a_offset);
160 a_offset_term_s32.val[3] = vmulq_n_s32(a_offset_term_s32.val[3], a_offset);
163 int32x4x4_t offset_term_s32 =
166 vdupq_n_s32(k_offset),
167 vdupq_n_s32(k_offset),
168 vdupq_n_s32(k_offset),
169 vdupq_n_s32(k_offset)
173 offset_term_s32.val[0] = vaddq_s32(offset_term_s32.val[0], vaddq_s32(a_offset_term_s32.val[0], b_offset_term_s32_vec));
174 offset_term_s32.val[1] = vaddq_s32(offset_term_s32.val[1], vaddq_s32(a_offset_term_s32.val[1], b_offset_term_s32_vec));
175 offset_term_s32.val[2] = vaddq_s32(offset_term_s32.val[2], vaddq_s32(a_offset_term_s32.val[2], b_offset_term_s32_vec));
176 offset_term_s32.val[3] = vaddq_s32(offset_term_s32.val[3], vaddq_s32(a_offset_term_s32.val[3], b_offset_term_s32_vec));
181 vld1q_s32(mm_result_ptr + x + 0),
182 vld1q_s32(mm_result_ptr + x + 4),
183 vld1q_s32(mm_result_ptr + x + 8),
184 vld1q_s32(mm_result_ptr + x + 12)
189 in_s32.val[0] = vaddq_s32(in_s32.val[0], offset_term_s32.val[0]);
190 in_s32.val[1] = vaddq_s32(in_s32.val[1], offset_term_s32.val[1]);
191 in_s32.val[2] = vaddq_s32(in_s32.val[2], offset_term_s32.val[2]);
192 in_s32.val[3] = vaddq_s32(in_s32.val[3], offset_term_s32.val[3]);
195 vst1q_s32(mm_result_ptr + x + 0, in_s32.val[0]);
196 vst1q_s32(mm_result_ptr + x + 4, in_s32.val[1]);
197 vst1q_s32(mm_result_ptr + x + 8, in_s32.val[2]);
198 vst1q_s32(mm_result_ptr + x + 12, in_s32.val[3]);
202 for(; x < window_end_x; ++x)
205 int32_t a_offset_term_s32 = *(vector_sum_col_ptr + x);
207 a_offset_term_s32 *= a_offset;
211 mm_result_ptr[x] += k_offset + a_offset_term_s32 + b_offset_term_s32;
214 vector_sum_col_it, vector_sum_row_it, mm_result_it);
216 else if((a_offset == 0) && (b_offset != 0) && (vector_sum_row !=
nullptr))
221 Window win_vector_sum_row(collapsed_window);
222 win_vector_sum_row.set(
Window::DimX, Window::Dimension(0, 0, 0));
223 win_vector_sum_row.set(
Window::DimY, Window::Dimension(0, 0, 0));
224 win_vector_sum_row.set(
Window::DimZ, Window::Dimension(0, 0, 0));
226 Iterator vector_sum_row_it(vector_sum_row, win_vector_sum_row);
228 const size_t sum_row_stride_y = vector_sum_row->info()->strides_in_bytes().y();
232 const int batch_id =
id.z() / depth_input;
233 auto mm_result_ptr =
reinterpret_cast<int32_t *
>(mm_result_it.ptr());
236 int32_t b_offset_term_s32 = *(
reinterpret_cast<const int32_t *
>(vector_sum_row_it.ptr() + batch_id * sum_row_stride_y) +
id.y() + (
id.z() % depth_input) * height_input);
237 b_offset_term_s32 *= b_offset;
239 const int32x4_t b_offset_term_s32_vec = vdupq_n_s32(b_offset_term_s32);
241 int x = window_start_x;
242 for(; x <= (window_end_x - window_step_x); x += window_step_x)
247 vld1q_s32(mm_result_ptr + x + 0),
248 vld1q_s32(mm_result_ptr + x + 4),
249 vld1q_s32(mm_result_ptr + x + 8),
250 vld1q_s32(mm_result_ptr + x + 12)
255 in_s32.val[0] = vaddq_s32(in_s32.val[0], b_offset_term_s32_vec);
256 in_s32.val[1] = vaddq_s32(in_s32.val[1], b_offset_term_s32_vec);
257 in_s32.val[2] = vaddq_s32(in_s32.val[2], b_offset_term_s32_vec);
258 in_s32.val[3] = vaddq_s32(in_s32.val[3], b_offset_term_s32_vec);
261 vst1q_s32(mm_result_ptr + x + 0, in_s32.val[0]);
262 vst1q_s32(mm_result_ptr + x + 4, in_s32.val[1]);
263 vst1q_s32(mm_result_ptr + x + 8, in_s32.val[2]);
264 vst1q_s32(mm_result_ptr + x + 12, in_s32.val[3]);
268 for(; x < window_end_x; ++x)
272 mm_result_ptr[x] += b_offset_term_s32;
275 vector_sum_row_it, mm_result_it);
277 else if((a_offset != 0) && (b_offset == 0) && (vector_sum_col !=
nullptr))
280 Window win_vector_sum_col(collapsed_window);
281 win_vector_sum_col.set(
Window::DimY, Window::Dimension(0, 0, 0));
282 win_vector_sum_col.set(
Window::DimZ, Window::Dimension(0, 0, 0));
284 Iterator vector_sum_col_it(vector_sum_col, win_vector_sum_col);
287 const int vector_sum_col_batch_offset = slide_vector_sum_col ? vector_sum_col->info()->strides_in_bytes().z() : 0;
291 const int batch_id =
id.z() / depth_input;
292 auto vector_sum_col_ptr =
reinterpret_cast<const int32_t *
>(vector_sum_col_it.ptr() + batch_id * vector_sum_col_batch_offset);
293 auto mm_result_ptr =
reinterpret_cast<int32_t *
>(mm_result_it.ptr());
295 int x = window_start_x;
296 for(; x <= (window_end_x - window_step_x); x += window_step_x)
299 int32x4x4_t a_offset_term_s32 =
302 vld1q_s32(vector_sum_col_ptr + x + 0),
303 vld1q_s32(vector_sum_col_ptr + x + 4),
304 vld1q_s32(vector_sum_col_ptr + x + 8),
305 vld1q_s32(vector_sum_col_ptr + x + 12)
309 a_offset_term_s32.val[0] = vmulq_n_s32(a_offset_term_s32.val[0], a_offset);
310 a_offset_term_s32.val[1] = vmulq_n_s32(a_offset_term_s32.val[1], a_offset);
311 a_offset_term_s32.val[2] = vmulq_n_s32(a_offset_term_s32.val[2], a_offset);
312 a_offset_term_s32.val[3] = vmulq_n_s32(a_offset_term_s32.val[3], a_offset);
317 vld1q_s32(mm_result_ptr + x + 0),
318 vld1q_s32(mm_result_ptr + x + 4),
319 vld1q_s32(mm_result_ptr + x + 8),
320 vld1q_s32(mm_result_ptr + x + 12)
325 in_s32.val[0] = vaddq_s32(in_s32.val[0], a_offset_term_s32.val[0]);
326 in_s32.val[1] = vaddq_s32(in_s32.val[1], a_offset_term_s32.val[1]);
327 in_s32.val[2] = vaddq_s32(in_s32.val[2], a_offset_term_s32.val[2]);
328 in_s32.val[3] = vaddq_s32(in_s32.val[3], a_offset_term_s32.val[3]);
331 vst1q_s32(mm_result_ptr + x + 0, in_s32.val[0]);
332 vst1q_s32(mm_result_ptr + x + 4, in_s32.val[1]);
333 vst1q_s32(mm_result_ptr + x + 8, in_s32.val[2]);
334 vst1q_s32(mm_result_ptr + x + 12, in_s32.val[3]);
338 for(; x < window_end_x; ++x)
341 const int32_t a_offset_term_s32 = *(vector_sum_col_ptr + x);
345 mm_result_ptr[x] += a_offset_term_s32 * a_offset;
348 vector_sum_col_it, mm_result_it);
359 : _vector_sum_col(nullptr), _vector_sum_row(nullptr), _mm_result(nullptr), _a_offset(0), _b_offset(0), _k_offset(0), _slide_vector_sum_col(true)
368 vector_sum_col !=
nullptr ? vector_sum_col->
info() :
nullptr,
369 vector_sum_row !=
nullptr ? vector_sum_row->
info() :
nullptr,
370 a_offset, b_offset));
372 _vector_sum_col = vector_sum_col;
373 _vector_sum_row = vector_sum_row;
374 _mm_result = mm_result;
375 _a_offset = a_offset;
376 _b_offset = b_offset;
377 _k_offset = a_offset * b_offset * k;
393 INEKernel::configure(win);
397 int32_t a_offset, int32_t b_offset)
411 const bool reinterpret_as_3d = _vector_sum_row !=
nullptr 415 run_offset_contribution(window, _mm_result, _vector_sum_col, _vector_sum_row, _a_offset, _b_offset, _k_offset, _slide_vector_sum_col, reinterpret_as_3d);
virtual size_t num_dimensions() const =0
The number of dimensions of the tensor (rank)
Window calculate_max_window(const ValidRegion &valid_region, const Steps &steps, bool skip_border, BorderSize border_size)
const Window & window() const
The maximum window the kernel can be executed on.
void run(const Window &window, const ThreadInfo &info) override
Execute the kernel on the passed window.
#define ARM_COMPUTE_RETURN_ON_ERROR(status)
Checks if a status contains an error and returns it.
Store the tensor's metadata.
#define ARM_COMPUTE_ERROR_THROW_ON(status)
void configure(ITensor *mm_result, const ITensor *vector_sum_col, const ITensor *vector_sum_row, int32_t k, int32_t a_offset, int32_t b_offset)
Initialise the kernel's input and output.
#define ARM_COMPUTE_RETURN_ERROR_ON(cond)
If the condition is true, an error is returned.
Interface for Neon tensor.
static Status validate(const ITensorInfo *mm_result, const ITensorInfo *vector_sum_col, const ITensorInfo *vector_sum_row, int32_t a_offset, int32_t b_offset)
Static function to check if given info will lead to a valid configuration of NEGEMMLowpOffsetContribu...
Copyright (c) 2017-2021 Arm Limited.
virtual void set_valid_region(const ValidRegion &valid_region)=0
Set the valid region of the tensor.
1 channel, 1 S32 per channel
T x() const
Alias to access the size of the first dimension.
static constexpr size_t DimX
Alias for dimension 0 also known as X dimension.
#define ARM_COMPUTE_UNUSED(...)
To avoid unused variables warnings.
void collapse_from(size_t start)
Collapse dimensions starting from a given point.
virtual const TensorShape & tensor_shape() const =0
Size for each dimension of the tensor.
NEGEMMLowpOffsetContributionKernel()
Constructor.
Class to describe a number of elements in each dimension.
virtual ITensorInfo * info() const =0
Interface to be implemented by the child class to return the tensor's metadata.
#define ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(k)
static constexpr size_t DimY
Alias for dimension 1 also known as Y dimension.
ScaleKernelInfo info(interpolation_policy, default_border_mode, PixelValue(), sampling_policy, false)
Information about executing thread and CPU.
static constexpr size_t DimZ
Alias for dimension 2 also known as Z dimension.
unsigned int num_dimensions() const
Returns the effective dimensionality of the tensor.
#define ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(t, c,...)
Status validate_arguments(const ITensorInfo *input, const ITensorInfo *bias, const ITensorInfo *output, const GEMMLowpOutputStageInfo *output_stage)
#define ARM_COMPUTE_RETURN_ERROR_ON_MSG(cond, msg)
If the condition is true, an error is returned.
#define ARM_COMPUTE_ERROR_ON_NULLPTR(...)
void execute_window_loop(const Window &w, L &&lambda_function, Ts &&... iterators)
Iterate through the passed window, automatically adjusting the iterators and calling the lambda_funct...
void set_num_dimensions(size_t num_dimensions)
Set number of dimensions.
T y() const
Alias to access the size of the second dimension.
Container for valid region of a window.
Describe a multidimensional execution window.
#define ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(f, s)