Compute Library
 22.11
ClGemmLowpMatrixMultiplyReshapedOnlyRhsMMULKernel.cpp
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2022 Arm Limited.
3  *
4  * SPDX-License-Identifier: MIT
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a copy
7  * of this software and associated documentation files (the "Software"), to
8  * deal in the Software without restriction, including without limitation the
9  * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10  * sell copies of the Software, and to permit persons to whom the Software is
11  * furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in all
14  * copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22  * SOFTWARE.
23  */
25 
30 
33 
34 #include "support/Cast.h"
35 
36 namespace arm_compute
37 {
38 namespace opencl
39 {
40 namespace kernels
41 {
42 using namespace misc::shape_calculator;
43 
44 namespace
45 {
46 using ElementsProcessed = Steps;
47 
48 Status validate_arguments(const ITensorInfo *src0, const ITensorInfo *src1, const ITensorInfo *dst, const GEMMKernelInfo &gemm_info,
49  const ITensorInfo *vector_sum_col, const ITensorInfo *vector_sum_row, const ITensorInfo *bias,
50  const ITensorInfo *output_multipliers, const ITensorInfo *output_shifts)
51 {
52  ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(src0, src1, dst);
53  ARM_COMPUTE_RETURN_ERROR_ON_MSG(!arm_matrix_multiply_supported(CLKernelLibrary::get().get_device()), "The extension cl_arm_matrix_multiply is not supported on the target platform");
56  ARM_COMPUTE_RETURN_ERROR_ON_MSG(src0->num_dimensions() > 4, "The number of dimensions for the LHS matrix must be <= 4");
57  ARM_COMPUTE_RETURN_ERROR_ON_MSG(src1->num_dimensions() > 3, "The number of dimensions for the RHS matrix must be <= 3");
58 
59  const GEMMRHSMatrixInfo rhs_info = gemm_info.rhs_info;
60  const GEMMLHSMatrixInfo lhs_info = gemm_info.lhs_info;
61  const GEMMLowpOutputStageInfo output_stage = gemm_info.output_stage;
62 
63  ARM_COMPUTE_RETURN_ERROR_ON_MSG(rhs_info.k0 != 4 || lhs_info.k0 != 4, "Only 4 is supported as value for k0");
64  ARM_COMPUTE_RETURN_ERROR_ON_MSG(!(lhs_info.m0 == 1 || lhs_info.m0 == 2 || lhs_info.m0 == 4), "Only 1,2,4 are supported for m0");
65  ARM_COMPUTE_RETURN_ERROR_ON_MSG(!(rhs_info.n0 == 1 || rhs_info.n0 == 4 || rhs_info.n0 == 8), "Only 1,4,8 are supported for n0");
66  ARM_COMPUTE_RETURN_ERROR_ON_MSG(rhs_info.export_to_cl_image, "Export to CLImage not supported for quantized GEMM");
67 
68  const int m = gemm_info.m;
69  const int n = gemm_info.n;
70  const int k = gemm_info.k;
71 
72  TensorShape tensor_shape1{ src1->tensor_shape() };
73  tensor_shape1.set(0, n);
74  tensor_shape1.set(1, k);
75 
76  const TensorInfo tensor_info1 = src1->clone()->set_tensor_shape(tensor_shape1);
77  const TensorInfo tensor_info_reshaped1 = src1->clone()->set_tensor_shape(compute_rhs_reshaped_shape(tensor_info1, rhs_info));
78 
79  ARM_COMPUTE_RETURN_ERROR_ON(src0->dimension(0) != static_cast<unsigned int>(k));
80  if(gemm_info.reinterpret_input_as_3d)
81  {
82  ARM_COMPUTE_RETURN_ERROR_ON(src0->dimension(1) * src0->dimension(2) != static_cast<unsigned int>(m));
83  }
84  else
85  {
86  ARM_COMPUTE_RETURN_ERROR_ON(src0->dimension(1) != static_cast<unsigned int>(m));
87  }
88  ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(src1, &tensor_info_reshaped1);
89 
90  const TensorShape expected_dst_shape = compute_mm_shape(*src0, *src1, gemm_info);
91  if(dst->total_size() != 0)
92  {
93  const TensorInfo tensor_info_dst = dst->clone()->set_tensor_shape(expected_dst_shape);
95  if(output_stage.type == GEMMLowpOutputStageType::NONE)
96  {
98  }
99  else
100  {
102  }
103  }
104 
105  if(bias != nullptr)
106  {
108  ARM_COMPUTE_RETURN_ERROR_ON(expected_dst_shape[0] != bias->dimension(0));
109  }
110 
112  "Only GEMMLowpOutputStageType::QUANTIZE_DOWN_FIXEDPOINT is supported");
113 
114  // Checks performed if the dst stage needs to be fused
116  {
117  // If a_offset == 0, vector_sum_col can be a nullptr
118  if(gemm_info.a_offset != 0)
119  {
121  ARM_COMPUTE_RETURN_ERROR_ON(vector_sum_col->dimension(0) != expected_dst_shape[0]);
122  }
123 
124  // If b_offset == 0, vector_sum_row can be a nullptr
125  if(gemm_info.b_offset != 0)
126  {
128 
129  // Check if mm result is a 3D reinterpretation
130  const bool reinterpret_as_3d = expected_dst_shape.num_dimensions() > 1 && expected_dst_shape.y() != vector_sum_row->tensor_shape().x();
131 
132  // Validate input
133  ARM_COMPUTE_RETURN_ERROR_ON(reinterpret_as_3d && vector_sum_row->dimension(0) != (expected_dst_shape[1] * expected_dst_shape[2]));
134  ARM_COMPUTE_RETURN_ERROR_ON(!reinterpret_as_3d && vector_sum_row->dimension(0) != expected_dst_shape[1]);
135 
136  if(expected_dst_shape.num_dimensions() > 1)
137  {
138  const unsigned int dst_batch_idx = reinterpret_as_3d ? 3 : 2;
139 
140  TensorShape vector_sum_row_shape = vector_sum_row->tensor_shape();
141  vector_sum_row_shape.collapse_from(1);
142  TensorShape collapsed_dst_shape(expected_dst_shape);
143  collapsed_dst_shape.collapse_from(dst_batch_idx);
144 
145  ARM_COMPUTE_RETURN_ERROR_ON_MSG(vector_sum_row_shape[1] != collapsed_dst_shape[dst_batch_idx],
146  "vector_sum_row must have the same number of batches of dst tensor");
147 
148  if(gemm_info.a_offset != 0)
149  {
150  TensorShape vector_sum_col_shape = vector_sum_col->tensor_shape();
151  vector_sum_col_shape.collapse_from(1);
152 
153  ARM_COMPUTE_RETURN_ERROR_ON_MSG(vector_sum_col_shape[1] != 1 && vector_sum_col_shape[1] != vector_sum_row_shape[1],
154  "vector_sum_col tensor must have the same number of batches of vector_sum_row_shape or the number of batches must be set to 1");
155  }
156  }
157  }
158 
159  if(dst->total_size() != 0)
160  {
161  ARM_COMPUTE_RETURN_ERROR_ON(output_stage.output_data_type != dst->data_type());
162  }
163  ARM_COMPUTE_RETURN_ERROR_ON(output_stage.gemmlowp_min_bound > output_stage.gemmlowp_max_bound);
164 
165  if(output_multipliers != nullptr && output_shifts != nullptr)
166  {
168  ARM_COMPUTE_RETURN_ERROR_ON(output_multipliers->num_dimensions() > 1);
170  ARM_COMPUTE_RETURN_ERROR_ON(output_shifts->num_dimensions() > 1);
171  if(output_stage.is_quantized_per_channel)
172  {
173  ARM_COMPUTE_RETURN_ERROR_ON(expected_dst_shape[0] != output_shifts->dimension(0));
174  ARM_COMPUTE_RETURN_ERROR_ON(expected_dst_shape[0] != output_multipliers->dimension(0));
175  }
176  }
177  }
178  return Status{};
179 }
180 
181 std::pair<Status, Window> validate_and_configure_window(const ITensorInfo *src0, const ITensorInfo *src1, ITensorInfo *dst, const GEMMKernelInfo &gemm_info,
182  ITensorInfo *vector_sum_col, const ITensorInfo *vector_sum_row, ITensorInfo *bias,
183  ITensorInfo *output_multipliers, ITensorInfo *output_shifts, ElementsProcessed &num_elements_processed)
184 {
185  const GEMMLowpOutputStageInfo output_stage = gemm_info.output_stage;
186 
187  unsigned int &num_elems_processed_per_iteration_x = num_elements_processed[0];
188  unsigned int &num_elems_processed_per_iteration_y = num_elements_processed[1];
189  bool reinterpret_output_as_3d = (gemm_info.depth_output_gemm3d != 0);
190 
191  Window win{};
192  bool window_changed = false;
193 
194  constexpr unsigned int mmul_n0 = 4;
195  constexpr unsigned int mmul_m0 = 4;
196  constexpr unsigned int mmul_k0 = 16;
197 
198  reinterpret_output_as_3d = false;
199  // dst tensor auto initialization if not yet initialized
200  const TensorShape expected_dst_shape = compute_mm_shape(*src0, *src1, gemm_info);
201  if(output_stage.type != GEMMLowpOutputStageType::NONE)
202  {
203  auto_init_if_empty(*dst, src0->clone()->set_tensor_shape(expected_dst_shape).set_data_type(output_stage.output_data_type));
204  }
205  else
206  {
207  auto_init_if_empty(*dst, src0->clone()->set_tensor_shape(expected_dst_shape).set_data_type(DataType::S32));
208  }
209 
210  TensorInfo tmp_info(*dst);
211 
212  if(reinterpret_output_as_3d)
213  {
214  // Since the dst tensor has to be reinterpreted as 3D and the execute window is based on a 2D GEMM,
215  // the window needs to be constructed on the 2D collapsed version of the tensor
216  TensorShape tmp_shape(dst->tensor_shape());
217  tmp_shape.collapse(2U, 1U);
218  tmp_info.set_tensor_shape(tmp_shape);
219  }
220 
221  // Configure kernel window
222  num_elems_processed_per_iteration_x = 1;
223  num_elems_processed_per_iteration_y = 1;
224 
225  win = calculate_max_window(tmp_info, Steps(num_elems_processed_per_iteration_x, num_elems_processed_per_iteration_y));
226 
228  {
229  if(gemm_info.a_offset != 0)
230  {
231  AccessWindowHorizontal vector_sum_col_access(vector_sum_col, 0, num_elems_processed_per_iteration_x);
232  window_changed = window_changed || update_window_and_padding(win, vector_sum_col_access);
233  }
234  // No access window needed for vector_sum_row
235  ARM_COMPUTE_UNUSED(vector_sum_row);
236 
237  if(bias != nullptr)
238  {
239  AccessWindowHorizontal bias_access(bias, 0, num_elems_processed_per_iteration_x);
240  window_changed = window_changed || update_window_and_padding(win, bias_access);
241  }
242 
243  if(output_multipliers != nullptr && output_stage.is_quantized_per_channel)
244  {
245  AccessWindowHorizontal output_multipliers_access(output_multipliers, 0, num_elems_processed_per_iteration_x);
246  AccessWindowHorizontal output_shifts_access(output_shifts, 0, num_elems_processed_per_iteration_x);
247  window_changed = window_changed || update_window_and_padding(win, output_multipliers_access, output_shifts_access);
248  }
249  }
250 
251  // Collapse along the Z direction
252  // This collapse needs to be here in order to tune the Z dimension of LWS
253  const unsigned int dimension_to_collapse = std::min(static_cast<unsigned int>(dst->num_dimensions()), 2u);
254  Window collapsed = win.collapse(win, dimension_to_collapse);
255 
256  // Reconfigure window size, one arm_matrix_multiply kernel needs 16 threads to finish.
257  Window::Dimension x_dimension = collapsed.x();
258  Window::Dimension y_dimension = collapsed.y();
259 
260  // Make M and N multiple of M0 and N0 respectively
261  const unsigned int ceil_to_multiple_n_n0 = ceil_to_multiple(x_dimension.end(), gemm_info.rhs_info.n0);
262  const unsigned int ceil_to_multiple_m_m0 = ceil_to_multiple(y_dimension.end(), gemm_info.lhs_info.m0);
263 
264  // Divide M and N by M0 and N0 respectively
265  const unsigned int n_div_n0 = ceil_to_multiple_n_n0 / gemm_info.rhs_info.n0;
266  const unsigned int m_div_m0 = ceil_to_multiple_m_m0 / gemm_info.lhs_info.m0;
267 
268  // Make n_div_n0 and m_div_m0 multiple of mmul_n0 and mmul_k0 respectively
269  const unsigned int ceil_to_multiple_n_div_n0_mmul_n0 = ceil_to_multiple(n_div_n0, mmul_n0);
270  const unsigned int ceil_to_multiple_m_div_m0_mmul_m0 = ceil_to_multiple(m_div_m0, mmul_k0);
271 
272  // Ensure x_dimension is multiple of MMUL block size (mmul_n0 * mmul_m0)
273  x_dimension.set_end(ceil_to_multiple_n_div_n0_mmul_n0 * mmul_n0);
274  y_dimension.set_end(ceil_to_multiple_m_div_m0_mmul_m0 / mmul_m0);
275 
276  collapsed.set(Window::DimX, x_dimension);
277  collapsed.set(Window::DimY, y_dimension);
278 
279  Status err = (window_changed) ? ARM_COMPUTE_CREATE_ERROR(ErrorCode::RUNTIME_ERROR, "Insufficient Padding!") : Status{};
280  return std::make_pair(err, collapsed);
281 }
282 } // namespace
283 
285 {
286  _type = CLKernelType::GEMM;
287 }
288 
290  const GEMMKernelInfo &gemm_info,
291  ITensorInfo *vector_sum_col, const ITensorInfo *vector_sum_row, ITensorInfo *bias,
292  ITensorInfo *output_multipliers, ITensorInfo *output_shifts)
293 {
294  ARM_COMPUTE_ERROR_ON_NULLPTR(src0, src1, dst);
295  ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(src0, src1, dst, gemm_info, vector_sum_col, vector_sum_row, bias, output_multipliers, output_shifts));
296 
297  auto padding_info = get_padding_info({ src0, src1, dst, vector_sum_row });
298  const GEMMRHSMatrixInfo rhs_info = gemm_info.rhs_info;
299  const GEMMLHSMatrixInfo lhs_info = gemm_info.lhs_info;
300  const GEMMLowpOutputStageInfo output_stage = gemm_info.output_stage;
301  const int32_t a_offset = gemm_info.a_offset;
302  const int32_t b_offset = gemm_info.b_offset;
303  constexpr int mmul_m0 = 4;
304  constexpr int mmul_n0 = 4;
305  constexpr int mmul_k0 = 16;
306 
307  _m = gemm_info.m;
308  _n = gemm_info.n;
309  _k = gemm_info.k;
310 
311  ElementsProcessed num_elements_processed{};
312 
313  // Configure kernel window
314  auto win_config = validate_and_configure_window(src0, src1, dst, gemm_info, vector_sum_col, vector_sum_row, bias, output_multipliers, output_shifts, num_elements_processed);
315  ARM_COMPUTE_ERROR_THROW_ON(win_config.first);
316  ICLKernel::configure_internal(win_config.second);
317 
318  const unsigned int m0_leftover = _m % lhs_info.m0;
319  const unsigned int n0_leftover = _n % rhs_info.n0;
320 
321  // Create build options
322  CLBuildOptions build_opts;
323  build_opts.add_option("-DDATA_TYPE=" + get_cl_type_from_data_type(src0->data_type()));
324  build_opts.add_option("-DVEC_TYPE=" + get_cl_type_from_data_type(src0->data_type()) + "4");
325  build_opts.add_option("-DACC_DATA_TYPE=int");
326  build_opts.add_option("-DOUT_DATA_TYPE=" + get_cl_type_from_data_type(dst->data_type()));
327  build_opts.add_option("-DM0=" + support::cpp11::to_string(lhs_info.m0));
328  build_opts.add_option("-DN0=" + support::cpp11::to_string(rhs_info.n0));
329  build_opts.add_option("-DK0=" + support::cpp11::to_string(rhs_info.k0));
330  build_opts.add_option("-DM0_LEFTOVER=" + support::cpp11::to_string(m0_leftover));
331  build_opts.add_option("-DN0_LEFTOVER=" + support::cpp11::to_string(n0_leftover));
332  build_opts.add_option("-DMMUL_M0=" + support::cpp11::to_string(mmul_m0));
333  build_opts.add_option("-DMMUL_N0=" + support::cpp11::to_string(mmul_n0));
334  build_opts.add_option("-DMMUL_K0=" + support::cpp11::to_string(mmul_k0));
335  build_opts.add_option("-DACTIVATION_TYPE=" + lower_string(string_from_activation_func(gemm_info.activation_info.activation())));
336  build_opts.add_option("-DA_VAL=" + float_to_string_with_full_precision(gemm_info.activation_info.a()));
337  build_opts.add_option("-DB_VAL=" + float_to_string_with_full_precision(gemm_info.activation_info.b()));
338 
339  std::string kernel_name("gemmlowp_mm_reshaped_only_rhs_mmul");
340 
342  {
343  build_opts.add_option("-DFUSED_OUTPUT_STAGE_FIXED_POINT");
344  _fuse_output_stage = true;
345  // If a_offset == 0, vector_sum_col can be a nullptr
346  if(a_offset != 0 && vector_sum_col != nullptr)
347  {
348  build_opts.add_option("-DA_OFFSET=" + support::cpp11::to_string(a_offset));
349  build_opts.add_option_if(vector_sum_col->tensor_shape().num_dimensions() > 1, "-DSUM_COL_HAS_BATCHES");
350  }
351  // If b_offset == 0, vector_sum_row can be a nullptr
352  build_opts.add_option_if(b_offset != 0, "-DB_OFFSET=" + support::cpp11::to_string(b_offset));
353  build_opts.add_option("-DK_OFFSET=" + support::cpp11::to_string(a_offset * b_offset * src0->dimension(0)));
354  build_opts.add_option_if(bias != nullptr, "-DADD_BIAS");
355  build_opts.add_option_if(gemm_info.broadcast_bias == true, "-DBROADCAST_BIAS");
356  build_opts.add_option("-DRESULT_OFFSET=" + support::cpp11::to_string(output_stage.gemmlowp_offset));
357  build_opts.add_option("-DRESULT_MULTIPLIER=" + support::cpp11::to_string(output_stage.gemmlowp_multipliers[0]));
358  build_opts.add_option("-DRESULT_SHIFT=" + support::cpp11::to_string(output_stage.gemmlowp_shifts[0]));
359 
360  const int min = output_stage.gemmlowp_min_bound;
361  const int max = output_stage.gemmlowp_max_bound;
362 
363  PixelValue min_val{};
364  PixelValue max_val{};
365  std::tie(min_val, max_val) = get_min_max(dst->data_type());
366  build_opts.add_option_if(min != min_val.get<int32_t>(), "-DMIN_BOUND=" + support::cpp11::to_string(min));
367  build_opts.add_option_if(max != max_val.get<int32_t>(), "-DMAX_BOUND=" + support::cpp11::to_string(max));
368  }
369 
370  // A macro guard to compile ONLY the kernel of interest
371  build_opts.add_option("-D" + upper_string(kernel_name));
372 
373  // Create kernel
374  _kernel = create_kernel(compile_context, kernel_name, build_opts.options());
375 
376  // Set config_id for enabling LWS tuning
377  _config_id = kernel_name;
378  _config_id += "_";
379  _config_id += (bias != nullptr ? "add_bias_" : "");
380  _config_id += (gemm_info.broadcast_bias ? "broadcast_bias_" : "");
381  _config_id += (gemm_info.activation_info.enabled() ? "fused_activation_" : "");
382  _config_id += lower_string(string_from_data_type(src0->data_type()));
383  _config_id += "_";
384  _config_id += support::cpp11::to_string(_m);
385  _config_id += "_";
386  _config_id += support::cpp11::to_string(_n);
387  _config_id += "_";
388  _config_id += support::cpp11::to_string(_k);
389  _config_id += "_";
390  _config_id += support::cpp11::to_string(lhs_info.m0);
391  _config_id += "_";
392  _config_id += support::cpp11::to_string(rhs_info.n0);
393 
395 }
396 
398  const ITensorInfo *vector_sum_col, const ITensorInfo *vector_sum_row, const ITensorInfo *bias,
399  const ITensorInfo *output_multipliers, const ITensorInfo *output_shifts)
400 {
401  ElementsProcessed num_elements_processed{};
402  ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments(src0, src1, dst, gemm_info, vector_sum_col, vector_sum_row, bias, output_multipliers, output_shifts));
404  src1->clone().get(),
405  dst->clone().get(),
406  gemm_info,
407  vector_sum_col != nullptr ? vector_sum_col->clone().get() : nullptr,
408  vector_sum_row != nullptr ? vector_sum_row->clone().get() : nullptr,
409  bias != nullptr ? bias->clone().get() : nullptr,
410  output_multipliers != nullptr ? output_multipliers->clone().get() : nullptr,
411  output_shifts != nullptr ? output_shifts->clone().get() : nullptr,
412  num_elements_processed)
413  .first);
414 
415  return Status{};
416 }
417 
418 void ClGemmLowpMatrixMultiplyReshapedOnlyRhsMMULKernel::run_op(ITensorPack &tensors, const Window &window, cl::CommandQueue &queue)
419 {
422 
423  const auto src0 = utils::cast::polymorphic_downcast<const ICLTensor *>(tensors.get_const_tensor(TensorType::ACL_SRC_0));
424  const auto src1 = utils::cast::polymorphic_downcast<const ICLTensor *>(tensors.get_const_tensor(TensorType::ACL_SRC_1));
425  const auto src2 = utils::cast::polymorphic_downcast<const ICLTensor *>(tensors.get_const_tensor(TensorType::ACL_SRC_2));
426  const auto vector_sum_col = utils::cast::polymorphic_downcast<const ICLTensor *>(tensors.get_const_tensor(TensorType::ACL_VEC_COL_SUM));
427  const auto vector_sum_row = utils::cast::polymorphic_downcast<const ICLTensor *>(tensors.get_const_tensor(TensorType::ACL_VEC_ROW_SUM));
428  auto dst = utils::cast::polymorphic_downcast<ICLTensor *>(tensors.get_tensor(TensorType::ACL_DST));
429 
430  ARM_COMPUTE_ERROR_ON_NULLPTR(src0, src1, dst);
431 
432  if(src1->info()->num_dimensions() < 3)
433  {
434  // The stride_z for matrix B must be zero if we do not slice
435  ARM_COMPUTE_ERROR_ON(src1->info()->strides_in_bytes()[3] != 0);
436  }
437 
438  cl::Image2D src1_image2d;
439 
441 
442  do
443  {
444  unsigned int idx = 0;
445 
446  add_3d_tensor_nhw_argument(idx, src0);
447  add_3d_tensor_nhw_argument(idx, src1);
448 
449  // Bias buffer (_add_bias == true)
450  if(src2 != nullptr)
451  {
452  add_3d_tensor_nhw_argument(idx, src2);
453  }
454  // dst buffer
455  add_3d_tensor_nhw_argument(idx, dst);
456 
457  // Pass m, n and k at runtime as signed ints, to ensure results of any subtraction they could be operand in, would still be signed.
458  _kernel.setArg<cl_int>(idx++, _m);
459  _kernel.setArg<cl_int>(idx++, _n);
460  _kernel.setArg<cl_int>(idx++, _k);
461 
462  if(_fuse_output_stage)
463  {
464  if(vector_sum_col != nullptr)
465  {
466  add_3d_tensor_nhw_argument(idx, vector_sum_col);
467  }
468  if(vector_sum_row != nullptr)
469  {
470  add_3d_tensor_nhw_argument(idx, vector_sum_row);
471  }
472  }
473 
474  enqueue(queue, *this, slice, cl::NDRange(32, 2), false);
475  }
476  while(window.slide_window_slice_3D(slice));
477 }
478 } // namespace kernels
479 } // namespace opencl
480 } // namespace arm_compute
Class describing the value of a pixel for any image format.
Definition: PixelValue.h:34
Window calculate_max_window(const ValidRegion &valid_region, const Steps &steps, bool skip_border, BorderSize border_size)
bool broadcast_bias
Flag used to broadcast the bias addition.
bool arm_matrix_multiply_supported(const cl::Device &device)
Helper function to check whether the cl_arm_matrix_multiply extension is supported.
Definition: CLHelpers.cpp:494
const Window & window() const
The maximum window the kernel can be executed on.
Definition: IKernel.cpp:28
Quantize using a fixed point multiplication.
Descriptor used by the GEMM kernels.
bool enabled() const
Check if initialised.
Definition: Types.h:1694
virtual size_t dimension(size_t index) const =0
Return the size of the requested dimension.
void enqueue(cl::CommandQueue &queue, ICLKernel &kernel, const Window &window, const cl::NDRange &lws_hint=CLKernelLibrary::get().default_ndrange(), bool use_dummy_work_items=false)
Add the kernel to the command queue with the given window.
Definition: ICLKernel.cpp:32
const StringSet & options() const
Gets the current options list set.
float a() const
Get the alpha value.
Definition: Types.h:1684
#define ARM_COMPUTE_RETURN_ON_ERROR(status)
Checks if a status contains an error and returns it.
Definition: Error.h:204
std::string to_string(T &&value)
Convert integer and float values to string.
virtual DataType data_type() const =0
Data type used for each element of the tensor.
TensorShape compute_mm_shape(const ITensorInfo &input0, const ITensorInfo &input1, bool is_interleaved_transposed, const GEMMReshapeInfo &reshape_info)
Calculate the matrix multiplication output shape of two tensors.
#define ARM_COMPUTE_ERROR_ON(cond)
If the condition is true then an error message is printed and an exception thrown.
Definition: Error.h:466
const std::string & string_from_activation_func(ActivationLayerInfo::ActivationFunction act)
Translates a given activation function to a string.
Definition: Utils.cpp:163
static CLKernelLibrary & get()
Access the KernelLibrary singleton.
GEMM LHS (Left Hand Side) matrix information.
Definition: Types.h:2303
Store the tensor&#39;s metadata.
Definition: ITensorInfo.h:40
#define ARM_COMPUTE_ERROR_THROW_ON(status)
Definition: Error.h:455
int32_t gemmlowp_offset
GEMMLowp output stage offset used for quantizing to QASYMM8.
Definition: Types.h:2290
Status class.
Definition: Error.h:52
ActivationLayerInfo activation_info
Activation function to perform after the matrix multiplication.
int32_t gemmlowp_max_bound
GEMMLowp max value used to saturate down the output result before converting back to QASYMM8...
Definition: Types.h:2294
std::string lower_string(const std::string &val)
Lower a given string.
Definition: Utils.cpp:353
Status validate_arguments(const ITensorInfo *src, const ITensorInfo *weights, const ITensorInfo *dst, const PadStrideInfo &conv_info)
#define ARM_COMPUTE_RETURN_ERROR_ON(cond)
If the condition is true, an error is returned.
Definition: Error.h:296
GEMMLowpOutputStageType type
GEMMLowp output stage type.
Definition: Types.h:2289
static Status validate(const ITensorInfo *src0, const ITensorInfo *src1, const ITensorInfo *dst, const GEMMKernelInfo &gemm_info, const ITensorInfo *vector_sum_col=nullptr, const ITensorInfo *vector_sum_row=nullptr, const ITensorInfo *bias=nullptr, const ITensorInfo *output_multipliers=nullptr, const ITensorInfo *output_shifts=nullptr)
Static function to check if given info will lead to a valid configuration.
GEMMLHSMatrixInfo lhs_info
LHS matrix information used to retrieve the number of rows processed by each thread.
Copyright (c) 2017-2022 Arm Limited.
std::vector< int32_t > gemmlowp_shifts
GEMMLowp output stage multiplier used for quantizing to QASYMM8.
Definition: Types.h:2296
#define ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(...)
Definition: Validate.h:159
1 channel, 1 S32 per channel
void add_option(std::string option)
Adds option to the existing build option list.
const OutputStage & output_stage
const ITensor * get_const_tensor(int id) const
Get constant tensor of a given id.
Definition: ITensorPack.cpp:54
unsigned int m
Number of LHS rows.
std::string upper_string(const std::string &val)
Raise a given string to upper case.
Definition: Utils.cpp:360
cl::Kernel create_kernel(const CLCompileContext &ctx, const std::string &kernel_name, const std::set< std::string > &build_opts=std::set< std::string >())
Creates an opencl kernel using a compile context.
Definition: CLHelpers.cpp:404
unsigned int n
Number of RHS columns.
const std::string & string_from_data_type(DataType dt)
Convert a data type identity into a string.
Definition: Utils.cpp:135
static constexpr size_t DimX
Alias for dimension 0 also known as X dimension.
Definition: Window.h:43
bool update_window_and_padding(Window &win, Ts &&... patterns)
Update window and padding size for each of the access patterns.
Definition: WindowHelpers.h:46
#define ARM_COMPUTE_UNUSED(...)
To avoid unused variables warnings.
Definition: Error.h:152
GEMM RHS (Right Hand Side) matrix information.
Definition: Types.h:2318
int32_t b_offset
Offset to be added to each element of the matrix B.
std::string float_to_string_with_full_precision(float val)
Create a string with the float in full precision.
Definition: Utils.h:1124
virtual const TensorShape & tensor_shape() const =0
Size for each dimension of the tensor.
auto ceil_to_multiple(S value, T divisor) -> decltype(((value+divisor - 1)/divisor) *divisor)
Computes the smallest number larger or equal to value that is a multiple of divisor.
Definition: Utils.h:71
quantized, asymmetric fixed-point 8-bit number unsigned
std::vector< int32_t > gemmlowp_multipliers
GEMMLowp output stage multiplier used for quantizing to QASYMM8.
Definition: Types.h:2295
std::string get_cl_type_from_data_type(const DataType &dt)
Translates a tensor data type to the appropriate OpenCL type.
Definition: CLHelpers.cpp:39
GEMMLowpOutputStageInfo output_stage
GEMMLowp output stage information.
TensorShape compute_rhs_reshaped_shape(const ITensorInfo &a, const GEMMRHSMatrixInfo &rhs_info)
Calculate the Right Hand Side matrix reshaped shape.
bool auto_init_if_empty(ITensorInfo &info, const TensorShape &shape, int num_channels, DataType data_type, QuantizationInfo quantization_info=QuantizationInfo())
Auto initialize the tensor info (shape, number of channels and data type) if the current assignment i...
virtual std::unique_ptr< T > clone() const =0
Provide a clone of the current object of class T.
GEMMLowp output stage info.
Definition: Types.h:2287
Quantize using a floating point multiplication.
void add_option_if(bool cond, std::string option)
Adds option if a given condition is true;.
bool slide_window_slice_3D(Window &slice) const
Slide the passed 3D window slice.
Definition: Window.h:349
Quantize using an integer multiplication.
#define ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(k)
Definition: Validate.h:915
bool has_padding_changed(const std::unordered_map< const ITensorInfo *, PaddingSize > &padding_map)
Check if the previously stored padding info has changed after configuring a kernel.
Definition: Utils.cpp:603
CLCompileContext class.
int32_t a_offset
Offset to be added to each element of the matrix A.
static constexpr size_t DimY
Alias for dimension 1 also known as Y dimension.
Definition: Window.h:45
void configure(const CLCompileContext &compile_context, const ITensorInfo *src0, const ITensorInfo *src1, ITensorInfo *dst, const GEMMKernelInfo &gemm_info, ITensorInfo *vector_sum_col=nullptr, const ITensorInfo *vector_sum_row=nullptr, ITensorInfo *bias=nullptr, ITensorInfo *output_multipliers=nullptr, ITensorInfo *output_shifts=nullptr)
Initialise the kernel&#39;s source and destination.
std::pair< Status, Window > validate_and_configure_window(ITensorInfo *src, ITensorInfo *dst)
ITensor * get_tensor(int id)
Get tensor of a given id from the pac.
Definition: ITensorPack.cpp:64
GEMMRHSMatrixInfo rhs_info
RHS matrix information used for reshaping the RHS matrix.
#define ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(...)
Definition: Validate.h:439
#define ARM_COMPUTE_CREATE_ERROR(error_code, msg)
Creates an error with a given message.
Definition: Error.h:159
unsigned int num_dimensions() const
Returns the effective dimensionality of the tensor.
Definition: Dimensions.h:143
#define ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(...)
Definition: Validate.h:541
#define ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(t, c,...)
Definition: Validate.h:788
std::unordered_map< const ITensorInfo *, PaddingSize > get_padding_info(std::initializer_list< const ITensorInfo *> infos)
Stores padding information before configuring a kernel.
Definition: Utils.cpp:588
unsigned int k
Number of LHS columns or RHS rows.
#define ARM_COMPUTE_RETURN_ERROR_ON_MSG(cond, msg)
If the condition is true, an error is returned.
Definition: Error.h:244
Tensor packing service.
Definition: ITensorPack.h:39
#define ARM_COMPUTE_ERROR_ON_NULLPTR(...)
Definition: Validate.h:157
unsigned int m0
Number of rows processed by the matrix multiplication.
Definition: Types.h:2310
ActivationFunction activation() const
Get the type of activation function.
Definition: Types.h:1679
float b() const
Get the beta value.
Definition: Types.h:1689
quantized, asymmetric fixed-point 8-bit number signed
int32_t gemmlowp_min_bound
GEMMLowp min value used to saturate down the output result before converting back to QASYMM8...
Definition: Types.h:2293
Window first_slice_window_3D() const
First 3D slice of the window.
Definition: Window.h:305
std::string kernel_name
std::tuple< PixelValue, PixelValue > get_min_max(DataType dt)
Compute the mininum and maximum values a data type can take.
Definition: Utils.h:564
Describe a multidimensional execution window.
Definition: Window.h:39
Convolution using GEMM.
void run_op(ITensorPack &tensors, const Window &window, cl::CommandQueue &queue) override
Enqueue the OpenCL kernel to process the given window on the passed OpenCL command queue...
#define ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(f, s)
Definition: Validate.h:201
SimpleTensor< T > slice(const SimpleTensor< T > &src, Coordinates starts, Coordinates ends)
const int32_t * bias