Compute Library
 22.02
cl_gemm.cpp
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2017-2021 Arm Limited.
3  *
4  * SPDX-License-Identifier: MIT
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a copy
7  * of this software and associated documentation files (the "Software"), to
8  * deal in the Software without restriction, including without limitation the
9  * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10  * sell copies of the Software, and to permit persons to whom the Software is
11  * furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in all
14  * copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22  * SOFTWARE.
23  */
24 #ifndef ARM_COMPUTE_CL /* Needed by Utils.cpp to handle OpenCL exceptions properly */
25 #error "This example needs to be built with -DARM_COMPUTE_CL"
26 #endif /* ARM_COMPUTE_CL */
27 
28 #include "arm_compute/core/Types.h"
29 #include "arm_compute/core/Utils.h"
35 #include "src/core/CL/kernels/CLDepthConvertLayerKernel.h"
37 #include "src/core/CL/kernels/CLGEMMLowpMatrixMultiplyNativeKernel.h"
38 #include "src/core/CL/kernels/CLGEMMLowpMatrixMultiplyReshapedOnlyRHSKernel.h"
39 #include "src/core/CL/kernels/CLGEMMLowpOffsetContributionKernel.h"
40 #include "src/core/CL/kernels/CLGEMMLowpOffsetContributionOutputStageKernel.h"
41 #include "src/core/CL/kernels/CLGEMMLowpReductionKernel.h"
42 #include "src/core/CL/kernels/CLGEMMMatrixMultiplyReshapedKernel.h"
43 #include "src/core/CL/kernels/CLGEMMMatrixMultiplyReshapedOnlyRHSKernel.h"
44 #include "src/core/CL/kernels/CLGEMMReshapeLHSMatrixKernel.h"
45 #include "src/core/CL/kernels/CLGEMMReshapeRHSMatrixKernel.h"
46 #include "src/core/CL/kernels/CLIm2ColKernel.h"
47 #include "src/core/CL/kernels/CLWeightsReshapeKernel.h"
48 #include "tests/AssetsLibrary.h"
49 #include "tests/CL/CLAccessor.h"
50 #include "tests/Globals.h"
51 #include "tests/IAccessor.h"
52 #include "tests/SimpleTensor.h"
56 
57 #include "utils/TypePrinter.h"
58 #include "utils/Utils.h"
61 
62 #include "ValidateExample.h"
63 
64 #include <cstdlib>
65 
66 using namespace arm_compute;
67 using namespace utils;
68 using namespace arm_compute::test;
69 using namespace arm_compute::test::validation;
70 
71 constexpr float abs_tolerance_f32(0.0001f); /**< F32 Absolute tolerance value for comparing reference's output against implementation's output for
72  * floating point data types in case using relative tolerance fails because of small values */
73 RelativeTolerance<float> tolerance_f32(0.001f); /**< F32 Tolerance value for comparing reference's output against implementation's output for floating point data types */
74 RelativeTolerance<half_float::half> tolerance_f16(half(0.2)); /**< F16 Tolerance value for comparing reference's output against implementation's output for floating point data types */
75 constexpr float tolerance_num_f16 = 0.02f; /**< F16 Tolerance number */
76 
77 namespace
78 {
79 class GEMMCommandLineOptions final
80 {
81 public:
82  explicit GEMMCommandLineOptions(CommandLineParser &parser) noexcept
83  : help(parser.add_option<ToggleOption>("help")),
84  add_bias(parser.add_option<ToggleOption>("add_bias")),
85  M(parser.add_option<SimpleOption<int>>("m", 7)),
86  N(parser.add_option<SimpleOption<int>>("n", 3)),
87  K(parser.add_option<SimpleOption<int>>("k", 5)),
88  B(parser.add_option<SimpleOption<int>>("b", 1)),
89  alpha(parser.add_option<SimpleOption<float>>("alpha", 1.f)),
90  beta(parser.add_option<SimpleOption<float>>("beta", 0.f)),
91  offset_src0(parser.add_option<SimpleOption<int>>("offset_i0", 10)),
92  offset_src1(parser.add_option<SimpleOption<int>>("offset_i1", 10)),
93  offset_dst(parser.add_option<SimpleOption<int>>("offset_o", 10)),
94  scale_src0(parser.add_option<SimpleOption<float>>("scale_i0", 1.f / 255)),
95  scale_src1(parser.add_option<SimpleOption<float>>("scale_i1", 1.f / 255)),
96  scale_dst(parser.add_option<SimpleOption<float>>("scale_o", 1.f / 255)),
97  data_type()
98  {
99  // Setup data type
100  const std::set<arm_compute::DataType> supported_data_types
101  {
105  };
106  data_type = parser.add_option<EnumOption<DataType>>("type", supported_data_types, DataType::F32);
107 
108  // Setup help strings
109  help->set_help("Show this help message");
110  add_bias->set_help("Add bias to the GEMM. Used when running in QASYMM8");
111  M->set_help("M value");
112  N->set_help("N value");
113  K->set_help("K value");
114  B->set_help("B value - number of batches");
115  alpha->set_help("Alpha value");
116  beta->set_help("Beta value");
117  offset_src0->set_help("Offset of first input. Used when running in QASYMM8");
118  offset_src1->set_help("Offset of second input. Used when running in QASYMM8");
119  offset_dst->set_help("Offset of output. Used when running in QASYMM8");
120  scale_src0->set_help("Scale of first input. Used when running in QASYMM8");
121  scale_src1->set_help("Scale of second input. Used when running in QASYMM8");
122  scale_dst->set_help("Scale of output. Used when running in QASYMM8");
123  data_type->set_help("Data type to use");
124  }
125  /** Prevent instances of this class from being copied (As this class contains pointers) */
126  GEMMCommandLineOptions(const GEMMCommandLineOptions &) = delete;
127  /** Prevent instances of this class from being copied (As this class contains pointers) */
128  GEMMCommandLineOptions &operator=(const GEMMCommandLineOptions &) = delete;
129  /** Allow instances of this class to be moved */
130  GEMMCommandLineOptions(GEMMCommandLineOptions &&) noexcept(true) = default;
131  /** Allow instances of this class to be moved */
132  GEMMCommandLineOptions &operator=(GEMMCommandLineOptions &&) noexcept(true) = default;
133  /** Default destructor */
134  ~GEMMCommandLineOptions() = default;
135 
136 public:
137  ToggleOption *help;
138  ToggleOption *add_bias;
139  SimpleOption<int> *M;
140  SimpleOption<int> *N;
141  SimpleOption<int> *K;
142  SimpleOption<int> *B;
143  SimpleOption<float> *alpha;
144  SimpleOption<float> *beta;
145  SimpleOption<int> *offset_src0;
146  SimpleOption<int> *offset_src1;
147  SimpleOption<int> *offset_dst;
148  SimpleOption<float> *scale_src0;
149  SimpleOption<float> *scale_src1;
150  SimpleOption<float> *scale_dst;
151  EnumOption<arm_compute::DataType> *data_type;
152 };
153 } // namespace
154 
155 class CLGEMMValidateExample : public ValidateExample
156 {
157 public:
158  bool do_setup(int argc, char **argv) override
159  {
161 
162  // Parse options
163  CommandLineParser parser;
164  GEMMCommandLineOptions gemm_options(parser);
165  parser.parse(argc, argv);
166 
167  // Print help
168  const bool print_help = gemm_options.help->is_set() ? gemm_options.help->value() : false;
169  if(print_help)
170  {
171  parser.print_help(argv[0]);
172  return false;
173  }
174 
175  // Consume parameters
176  consume_params(gemm_options);
177  print_parameters_internal();
178 
179  const bool is_quantized = is_data_type_quantized(data_type);
180 
181  // Calculate re-quantization parameters
182  if(is_quantized)
183  {
184  float multiplier = scale_src0 * scale_src1 / scale_dst;
185  quantization::calculate_quantized_multiplier(multiplier, &dst_multiplier, &dst_shift);
186  }
187 
188  // Initialize GEMM inputs/outputs
189  src0.allocator()->init(TensorInfo(TensorShape(K, M, B), 1, data_type));
190  src1.allocator()->init(TensorInfo(TensorShape(N, K, B), 1, data_type));
191  src2.allocator()->init(TensorInfo(TensorShape(N, M, B), 1, data_type));
192  init_sgemm_output(dst, src0, src1, data_type);
193 
194  // Configure function
195  if(is_quantized)
196  {
197  src0.info()->set_quantization_info(QuantizationInfo(scale_src0, offset_src0));
198  src1.info()->set_quantization_info(QuantizationInfo(scale_src1, offset_src1));
199  dst.info()->set_quantization_info(QuantizationInfo(scale_dst, offset_dst));
200  biases.allocator()->init(TensorInfo(TensorShape(N), 1, DataType::S32));
201  init_sgemm_output(tmp_dst, src0, src1, DataType::S32);
202 
203  // Configure GEMMlowp matrix multiply function
204  mm_gemmlowp.configure(&src0, &src1, nullptr, &tmp_dst);
205 
206  // Configure GEMMlowp output stage
207  mm_gemmlowp_output_stage.configure(&tmp_dst, add_bias ? &biases : nullptr, &dst, dst_multiplier, dst_shift, offset_dst);
208  tmp_dst.allocator()->allocate();
209  biases.allocator()->allocate();
210  fill(CLAccessor(biases), 3);
211  }
212  else
213  {
214  // Configure matrix multiply function
215  mm_gemm.configure(&src0, &src1, &src2, &dst, alpha, beta);
216  }
217 
218  // Allocate all the tensors
219  src0.allocator()->allocate();
220  src1.allocator()->allocate();
221  dst.allocator()->allocate();
222  src2.allocator()->allocate();
223 
224  fill(CLAccessor(src0), 0);
225  fill(CLAccessor(src1), 1);
226  fill(CLAccessor(src2), 2);
227 
228  return true;
229  }
230 
231  void print_parameters_internal()
232  {
233  std::cout << "Datatype : " << string_from_data_type(data_type) << "\n";
234  std::cout << "M : " << support::cpp11::to_string(M) << "\n";
235  std::cout << "N : " << support::cpp11::to_string(N) << "\n";
236  std::cout << "K : " << support::cpp11::to_string(K) << "\n";
237  std::cout << "B : " << support::cpp11::to_string(B) << "\n";
239  {
240  std::cout << "Scale_Src0 : " << support::cpp11::to_string(scale_src0) << "\n";
241  std::cout << "Offset_Src0 : " << support::cpp11::to_string(offset_src0) << "\n";
242  std::cout << "Scale_Scr1 : " << support::cpp11::to_string(scale_src1) << "\n";
243  std::cout << "Offset_Src1 : " << support::cpp11::to_string(offset_src1) << "\n";
244  std::cout << "Scale_Dst : " << support::cpp11::to_string(scale_dst) << "\n";
245  std::cout << "Offset_Dst : " << support::cpp11::to_string(offset_dst) << "\n";
246  std::cout << "Bias : " << support::cpp11::to_string(add_bias) << "\n";
247  }
248  else
249  {
250  std::cout << "Alpha : " << support::cpp11::to_string(alpha) << "\n";
251  std::cout << "Beta : " << support::cpp11::to_string(beta) << "\n";
252  }
253  }
254 
255  void do_validate() override
256  {
257  switch(data_type)
258  {
259  case DataType::F16:
260  {
261  SimpleTensor<half> ref_src0 = { TensorShape(K, M, B), data_type, 1 };
262  SimpleTensor<half> ref_src1 = { TensorShape(N, K, B), data_type, 1 };
263  SimpleTensor<half> ref_src2 = { TensorShape(N, M, B), data_type, 1 };
264 
265  fill(ref_src0, 0);
266  fill(ref_src1, 1);
267  fill(ref_src2, 2);
268 
269  SimpleTensor<half> ref_dst = reference::gemm<half>(ref_src0, ref_src1, ref_src2, alpha, beta);
270  validate(CLAccessor(dst), ref_dst, tolerance_f16, tolerance_num_f16);
271  break;
272  }
273  case DataType::F32:
274  {
275  SimpleTensor<float> ref_src0 = { TensorShape(K, M, B), data_type, 1 };
276  SimpleTensor<float> ref_src1 = { TensorShape(N, K, B), data_type, 1 };
277  SimpleTensor<float> ref_src2 = { TensorShape(N, M, B), data_type, 1 };
278 
279  fill(ref_src0, 0);
280  fill(ref_src1, 1);
281  fill(ref_src2, 2);
282 
283  SimpleTensor<float> ref_dst = reference::gemm<float>(ref_src0, ref_src1, ref_src2, alpha, beta);
285  break;
286  }
287  case DataType::QASYMM8:
288  {
289  SimpleTensor<uint8_t> ref_src0{ TensorShape(K, M, B), data_type, 1 };
290  SimpleTensor<uint8_t> ref_src1{ TensorShape(N, K, B), data_type, 1 };
291  SimpleTensor<uint8_t> ref_dst;
292 
293  // Fill reference
294  fill(ref_src0, 0);
295  fill(ref_src1, 1);
296 
297  SimpleTensor<int32_t> ref_tmp_dst = reference::gemmlowp_matrix_multiply_core<int32_t, uint8_t>(ref_src0, ref_src1, TensorShape(N, M, B), offset_src0, offset_src1);
298 
299  const std::vector<int32_t> dst_multiplier_vec = { dst_multiplier };
300  const std::vector<int32_t> dst_shift_vec = { dst_shift };
301 
302  if(add_bias)
303  {
305  // Fill bias
306  fill(biases, 3);
307  ref_dst = reference::gemmlowp_quantize_down_scale_by_fixedpoint<int32_t, uint8_t>(ref_tmp_dst, biases, dst_multiplier_vec, dst_shift_vec, offset_dst);
308  }
309  else
310  {
311  ref_dst = reference::gemmlowp_quantize_down_scale_by_fixedpoint<int32_t, uint8_t>(ref_tmp_dst, dst_multiplier_vec, dst_shift_vec, offset_dst);
312  }
313  validate(CLAccessor(dst), ref_dst);
314  break;
315  }
316  default:
317  break;
318  }
319  }
320  void do_run() override
321  {
322  // Execute the function
324  {
325  // Run gemmlowp
326  mm_gemmlowp.run();
327  // Run output stage
328  mm_gemmlowp_output_stage.run();
329  }
330  else
331  {
332  // Run gemm
333  mm_gemm.run();
334  }
335 
336  // Make sure all the OpenCL jobs are done executing:
338  }
339 
340 private:
341  template <typename U>
342  void fill(U &&tensor, int i)
343  {
344  switch(tensor.data_type())
345  {
346  case DataType::F16:
347  {
349  library->fill(tensor, distribution, i);
350  break;
351  }
352  case DataType::F32:
353  {
354  std::uniform_real_distribution<float> distribution(-1.0f, 1.0f);
355  library->fill(tensor, distribution, i);
356  break;
357  }
358  case DataType::S32:
359  case DataType::QASYMM8:
360  {
361  std::uniform_int_distribution<> distribution(-6000, 6000);
362  library->fill(tensor, distribution, i);
363  break;
364  }
365  default:
366  library->fill_tensor_uniform(tensor, i);
367  }
368  }
369 
370  void consume_params(const GEMMCommandLineOptions &opts)
371  {
372  ARM_COMPUTE_ERROR_ON(opts.M->value() <= 0);
373  ARM_COMPUTE_ERROR_ON(opts.N->value() <= 0);
374  ARM_COMPUTE_ERROR_ON(opts.K->value() <= 0);
375  ARM_COMPUTE_ERROR_ON(opts.B->value() <= 0);
376  M = opts.M->value();
377  N = opts.N->value();
378  K = opts.K->value();
379  B = opts.B->value();
380  alpha = opts.alpha->value();
381  beta = opts.beta->value();
382  offset_src0 = opts.offset_src0->value();
383  offset_src1 = opts.offset_src1->value();
384  offset_dst = opts.offset_dst->value();
385  scale_src0 = opts.scale_src0->value();
386  scale_src1 = opts.scale_src1->value();
387  scale_dst = opts.scale_dst->value();
388  add_bias = opts.add_bias->is_set() ? opts.add_bias->value() : true;
389  data_type = opts.data_type->value();
390  }
391 
392  CLTensor src0{}, src1{}, src2{}, dst{};
393  CLTensor tmp_dst{}, biases{};
394 
395  CLGEMM mm_gemm{};
396  CLGEMMLowpMatrixMultiplyCore mm_gemmlowp{};
397  CLGEMMLowpQuantizeDownInt32ToUint8ScaleByFixedPoint mm_gemmlowp_output_stage{};
398 
399  size_t M{ 7 }, N{ 3 }, K{ 5 }, B{ 1 };
401  float alpha{ 1.0 }, beta{ 0.0 };
402  int offset_src0{ 10 }, offset_src1{ 10 }, offset_dst{ 10 };
403  float scale_src0{ 1.0f / 255 }, scale_src1{ 1.0f / 255 }, scale_dst{ 1.0f / 255 };
404  int32_t dst_multiplier{ 0 }, dst_shift{ 0 };
405  bool add_bias{ true };
406 };
407 
408 /** Main program for gemm test
409  *
410  * @param[in] argc Number of arguments
411  * @param[in] argv Arguments
412  *
413  */
414 int main(int argc, char **argv)
415 {
416  return utils::run_example<CLGEMMValidateExample>(argc, argv);
417 }
bool is_data_type_quantized(DataType dt)
Check if a given data type is of quantized type.
Definition: Utils.h:996
Shape of a tensor.
Definition: TensorShape.h:39
RelativeTolerance< float > tolerance_f32(0.001f)
F32 Tolerance value for comparing reference&#39;s output against implementation&#39;s output for floating poi...
constexpr float tolerance_num_f16
F16 Tolerance number.
Definition: cl_gemm.cpp:75
static CLScheduler & get()
Access the scheduler singleton.
std::string to_string(T &&value)
Convert integer and float values to string.
void default_init(ICLTuner *cl_tuner=nullptr, CLGEMMHeuristicsHandle *gemm_h=nullptr, CLBackendType cl_backend_type=CLBackendType::Native)
Initialises the context and command queue used by the scheduler to default values and sets a default ...
int main(int argc, char **argv)
Main program for gemm test.
Definition: cl_gemm.cpp:414
half_float::half half
16-bit floating point type
Definition: Types.h:48
1 channel, 1 F32 per channel
#define ARM_COMPUTE_ERROR_ON(cond)
If the condition is true then an error message is printed and an exception thrown.
Definition: Error.h:466
unsigned int M
Status calculate_quantized_multiplier(float multiplier, int32_t *quant_multiplier, int32_t *shift, bool ignore_epsilon=false)
Calculate quantized representation of multiplier.
Copyright (c) 2017-2021 Arm Limited.
1 channel, 1 F16 per channel
1 channel, 1 S32 per channel
Specialized class to generate random non-zero FP16 values.
Definition: Utils.h:254
const DataType data_type
Definition: Im2Col.cpp:150
Interface to enqueue OpenCL kernels and get/set the OpenCL CommandQueue and ICLTuner.
Quantization information.
const std::string & string_from_data_type(DataType dt)
Convert a data type identity into a string.
Definition: Utils.cpp:135
library fill(src, distribution, 0)
std::unique_ptr< AssetsLibrary > library
Definition: main.cpp:76
Basic function to execute GEMM on OpenCL.
Definition: CLGEMM.h:44
quantized, asymmetric fixed-point 8-bit number unsigned
Accessor implementation for CLTensor objects.
Definition: CLAccessor.h:36
unsigned int N
std::uniform_real_distribution< float > distribution(-5.f, 5.f)
void sync()
Blocks until all commands in the associated command queue have finished.
Definition: CLScheduler.cpp:70
Simple tensor object that stores elements in a consecutive chunk of memory.
Definition: SimpleTensor.h:58
Class reprensenting a relative tolerance value.
Definition: Validation.h:97
Store the tensor&#39;s metadata.
Definition: TensorInfo.h:43
Basic function to execute GEMMLowpMatrixMultiplyCore on OpenCL.
RelativeTolerance< half_float::half > tolerance_f16(half(0.2))
F16 Tolerance value for comparing reference&#39;s output against implementation&#39;s output for floating poi...
DataType
Available data types.
Definition: Types.h:79
void init_sgemm_output(T &dst, T &src0, T &src1, arm_compute::DataType dt)
Definition: Utils.h:764
constexpr float abs_tolerance_f32(0.0001f)
F32 Absolute tolerance value for comparing reference&#39;s output against implementation&#39;s output for flo...
Status validate(const ITensorInfo *scores_in, const ITensorInfo *boxes_in, const ITensorInfo *batch_splits_in, const ITensorInfo *scores_out, const ITensorInfo *boxes_out, const ITensorInfo *classes, const ITensorInfo *batch_splits_out, const ITensorInfo *keeps, const ITensorInfo *keeps_size, const BoxNMSLimitInfo info)
Basic implementation of the OpenCL tensor interface.
Definition: CLTensor.h:41
unsigned int K