Compute Library
 22.11
mergeresults.cpp
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2017-2021 Arm Limited.
3  *
4  * SPDX-License-Identifier: MIT
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a copy
7  * of this software and associated documentation files (the "Software"), to
8  * deal in the Software without restriction, including without limitation the
9  * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10  * sell copies of the Software, and to permit persons to whom the Software is
11  * furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in all
14  * copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22  * SOFTWARE.
23  */
24 
25 /* As some of the merges need these headers, but are all included in the
26  * arm_gemm namespace, put these headers here. */
27 #include <algorithm>
28 
29 #include <arm_neon.h>
30 
31 #include "arm_gemm.hpp"
32 #include "asmlib.hpp"
33 #include "utils.hpp"
34 
35 namespace arm_gemm {
36 
37 template<unsigned int twidth, unsigned int height, bool sve=false, typename Tin, typename Tout>
38 void MergeResults(Tout * out, const Tin * in, int ldc, int y0, int ymax, int x0, int xmax, const Tout *bias, Activation act, bool append) {
39  // NOTE: The following code is disabled to avoid calling get_vector_length(), so templated MergeResults will not
40  // be correct for SVE cases. This is OK as we have specialisations for all needed SVE cases anyway.
41  //
42  // For SVE cases, multiply the width up by the vector length.
43  // Use the *input* type to determine this, since this will be what the kernel operated on.
44  // const int width = twidth * (sve ? get_vector_length<Tin>() : 1);
45  const int width = twidth;
46 
47  const int full_y_blocks = (ymax - y0) / height;
48  const int y_remainder = (ymax - y0) % height;
49  const int y_blocks = full_y_blocks + (y_remainder ? 1 : 0);
50 
51  const int full_x_blocks = (xmax - x0) / width;
52  const int x_remainder = (xmax - x0) % width;
53  const int x_blocks = full_x_blocks + (x_remainder ? 1 : 0);
54 
55  for (int y_block = 0; y_block < y_blocks; y_block++) {
56  int ybase = y0 + (y_block * height);
57 
58  int fill_rows = (y_block < full_y_blocks) ? height : y_remainder;
59 
60  for (int x_block = 0; x_block < x_blocks; x_block++) {
61  int xbase = x0 + (x_block * width);
62 
63  int fill_cols = (x_block < full_x_blocks) ? width : x_remainder;
64 
65  for (int row=0; row < fill_rows; row++) {
66  for (int col=0; col < fill_cols; col++) {
67  Tout &r = out[(ybase + row) * ldc + xbase + col];
68  Tout v = in[row * width + col];
69 
70  if (append) {
71  v += r;
72  }
73 
74  if (bias) {
75  v += bias[xbase + col];
76  }
77 
78  switch(act.type) {
79  default:
81  break;
82 
84  v = std::max(v, static_cast<Tout>(0));
85  break;
86 
88  v = std::max(std::min(v, static_cast<Tout>(act.param1)), static_cast<Tout>(0));
89  break;
90  }
91 
92  r = v;
93  }
94  }
95 
96  in += (width * height);
97  }
98  }
99 }
100 
101 #include "merges/list.hpp"
102 
103 /* Cortex-A53 8x6 SGEMM kernel uses a templated merge as the optimized merge
104  * generator cannot cope with the width (6) not being a multiple of VL (4). */
105 #ifdef __aarch64__
106 template void MergeResults<6u, 8u, false, float, float>(float *, float const*, int, int, int, int, int, float const *, Activation, bool);
107 #endif
108 
109 #if defined(__aarch64__) && defined(__ARM_FP16_ARGS)
110 template void MergeResults<12u, 8u, false, float, __fp16>(__fp16*, float const*, int, int, int, int, int, __fp16 const*, Activation, bool);
111 #endif
112 
113 #if defined(__arm__) && defined(__ARM_FP16_ARGS)
114 template void MergeResults<8u, 6u, false, float, __fp16>(__fp16*, float const*, int, int, int, int, int, __fp16 const*, Activation, bool);
115 #endif
116 
117 } // namespace arm_gemm
void MergeResults(Tout *out, const Tin *in, int ldc, int y0, int ymax, int x0, int xmax, const Tout *bias, Activation act, bool append)
const int32_t * bias