24.02.1
generic.cpp
Go to the documentation of this file.
1
/*
2
* Copyright (c) 2021, 2023 Arm Limited.
3
*
4
* SPDX-License-Identifier: MIT
5
*
6
* Permission is hereby granted, free of charge, to any person obtaining a copy
7
* of this software and associated documentation files (the "Software"), to
8
* deal in the Software without restriction, including without limitation the
9
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10
* sell copies of the Software, and to permit persons to whom the Software is
11
* furnished to do so, subject to the following conditions:
12
*
13
* The above copyright notice and this permission notice shall be included in all
14
* copies or substantial portions of the Software.
15
*
16
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22
* SOFTWARE.
23
*/
24
25
#include <cstddef>
26
#include <cstdint>
27
28
#if defined(ARM_COMPUTE_ENABLE_SVE)
29
30
namespace
arm_conv
{
31
namespace
depthwise
{
32
33
void
sve_fp32_packed_to_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst_impl(
34
const
float
*
const
*
const
inptrs,
35
float
*
const
*
const
outptrs,
36
const
void
*params,
37
const
unsigned
int
n_output_channels,
38
const
float
activation_min
,
39
const
float
activation_max
40
)
41
{
42
const
float
minmax_vals[2] = {
activation_min
,
activation_max
};
43
44
__asm__ __volatile__(
45
"mov x17, #0x0\n"
46
"whilelt p2.s, x17, %x[channel_multiplier]\n"
47
"ldr x16, [%x[inptrs], #0x0]\n"
48
"ldr x15, [%x[inptrs], #0x8]\n"
49
"ptrue p1.b\n"
50
"ldr x14, [%x[inptrs], #0x10]\n"
51
"ldr x13, [%x[inptrs], #0x18]\n"
52
"mov x12, #0x0\n"
53
"ldr x11, [%x[inptrs], #0x20]\n"
54
"ldr x10, [%x[inptrs], #0x28]\n"
55
"ldr x9, [%x[inptrs], #0x30]\n"
56
"ld1w { z24.s }, p2/Z, [%x[params]]\n"
57
"mov z21.d, z24.d\n"
58
"mov z25.d, z24.d\n"
59
"ldp x28, x27, [%x[outptrs], #0x0]\n"
60
"ldp x26, x25, [%x[outptrs], #0x10]\n"
61
"mov z27.d, z24.d\n"
62
"mov z26.d, z24.d\n"
63
"ldp x24, x23, [%x[outptrs], #0x20]\n"
64
"ldp x22, x21, [%x[outptrs], #0x30]\n"
65
"mov z28.d, z24.d\n"
66
"mov z20.d, z24.d\n"
67
"ldr x20, [%x[outptrs], #0x40]\n"
68
"ld1rqw { z2.s }, p1/Z, [x16]\n"
69
"mov z23.d, z24.d\n"
70
"mov z19.d, z24.d\n"
71
"ld1rqw { z3.s }, p1/Z, [x16, #16]\n"
72
"ld1rqw { z4.s }, p1/Z, [x15]\n"
73
"ld1rqw { z5.s }, p1/Z, [x15, #16]\n"
74
"ld1rqw { z6.s }, p1/Z, [x14]\n"
75
"ld1rqw { z7.s }, p1/Z, [x14, #16]\n"
76
"ld1rqw { z8.s }, p1/Z, [x13]\n"
77
"ld1rqw { z9.s }, p1/Z, [x13, #16]\n"
78
"ld1rqw { z10.s }, p1/Z, [x11]\n"
79
"ld1rqw { z11.s }, p1/Z, [x11, #16]\n"
80
"ld1rqw { z12.s }, p1/Z, [x10]\n"
81
"ld1rqw { z13.s }, p1/Z, [x10, #16]\n"
82
"ld1rqw { z14.s }, p1/Z, [x9]\n"
83
"ld1rqw { z15.s }, p1/Z, [x9, #16]\n"
84
"ld1rw { z22.s }, p1/Z, [%x[clamps]]\n"
85
"ld1rw { z16.s }, p1/Z, [%x[clamps], #4]\n"
86
"ld1w { z31.s }, p2/Z, [%x[params], #1, MUL VL]\n"
87
"ld1w { z30.s }, p2/Z, [%x[params], #2, MUL VL]\n"
88
"ld1w { z29.s }, p2/Z, [%x[params], #3, MUL VL]\n"
89
"addvl %x[params], %x[params], #4\n"
90
"1:"
// Output channel complete vector loop
91
"fmla z24.s, z31.s, z2.s[0]\n"
92
"fmla z27.s, z31.s, z6.s[0]\n"
93
"mov z1.d, z10.d\n"
94
"incw x17\n"
95
"fmla z26.s, z31.s, z6.s[2]\n"
96
"fmla z28.s, z31.s, z7.s[0]\n"
97
"mov z0.d, z11.d\n"
98
"mov p0.b, p2.b\n"
99
"fmla z21.s, z31.s, z2.s[2]\n"
100
"fmla z25.s, z31.s, z3.s[0]\n"
101
"whilelt p2.s, x17, %x[channel_multiplier]\n"
102
"fmla z20.s, z31.s, z1.s[0]\n"
103
"fmla z23.s, z31.s, z1.s[2]\n"
104
"fmla z19.s, z31.s, z0.s[0]\n"
105
"fmla z24.s, z30.s, z2.s[1]\n"
106
"ld1w { z18.s }, p1/Z, [%x[params]]\n"
107
"fmla z27.s, z30.s, z6.s[1]\n"
108
"fmla z26.s, z30.s, z6.s[3]\n"
109
"fmla z28.s, z30.s, z7.s[1]\n"
110
"fmla z21.s, z30.s, z2.s[3]\n"
111
"fmla z25.s, z30.s, z3.s[1]\n"
112
"fmla z20.s, z30.s, z1.s[1]\n"
113
"fmla z23.s, z30.s, z1.s[3]\n"
114
"fmla z19.s, z30.s, z0.s[1]\n"
115
"ld1w { z17.s }, p1/Z, [%x[params], #1, MUL VL]\n"
116
"fmla z24.s, z29.s, z2.s[2]\n"
117
"fmla z27.s, z29.s, z6.s[2]\n"
118
"fmla z26.s, z29.s, z7.s[0]\n"
119
"fmla z28.s, z29.s, z7.s[2]\n"
120
"fmla z21.s, z29.s, z3.s[0]\n"
121
"fmla z25.s, z29.s, z3.s[2]\n"
122
"fmla z20.s, z29.s, z1.s[2]\n"
123
"fmla z23.s, z29.s, z0.s[0]\n"
124
"mov z1.d, z8.d\n"
125
"fmla z19.s, z29.s, z0.s[2]\n"
126
"mov z0.d, z9.d\n"
127
"fmla z24.s, z18.s, z4.s[0]\n"
128
"ld1w { z31.s }, p1/Z, [%x[params], #2, MUL VL]\n"
129
"fmla z27.s, z18.s, z1.s[0]\n"
130
"fmla z26.s, z18.s, z1.s[2]\n"
131
"mov z1.d, z12.d\n"
132
"fmla z28.s, z18.s, z0.s[0]\n"
133
"mov z0.d, z13.d\n"
134
"fmla z21.s, z18.s, z4.s[2]\n"
135
"fmla z25.s, z18.s, z5.s[0]\n"
136
"fmla z20.s, z18.s, z1.s[0]\n"
137
"fmla z23.s, z18.s, z1.s[2]\n"
138
"fmla z19.s, z18.s, z0.s[0]\n"
139
"mov z1.d, z8.d\n"
140
"ld1w { z18.s }, p1/Z, [%x[params], #3, MUL VL]\n"
141
"mov z0.d, z9.d\n"
142
"fmla z24.s, z17.s, z4.s[1]\n"
143
"fmla z27.s, z17.s, z1.s[1]\n"
144
"fmla z26.s, z17.s, z1.s[3]\n"
145
"fmla z28.s, z17.s, z0.s[1]\n"
146
"mov z1.d, z12.d\n"
147
"mov z0.d, z13.d\n"
148
"fmla z21.s, z17.s, z4.s[3]\n"
149
"fmla z25.s, z17.s, z5.s[1]\n"
150
"fmla z20.s, z17.s, z1.s[1]\n"
151
"fmla z23.s, z17.s, z1.s[3]\n"
152
"mov z1.d, z8.d\n"
153
"fmla z19.s, z17.s, z0.s[1]\n"
154
"mov z0.d, z9.d\n"
155
"fmla z24.s, z31.s, z4.s[2]\n"
156
"ld1w { z17.s }, p1/Z, [%x[params], #4, MUL VL]\n"
157
"fmla z27.s, z31.s, z1.s[2]\n"
158
"fmla z26.s, z31.s, z0.s[0]\n"
159
"mov z1.d, z12.d\n"
160
"fmla z28.s, z31.s, z0.s[2]\n"
161
"mov z0.d, z13.d\n"
162
"fmla z21.s, z31.s, z5.s[0]\n"
163
"fmla z25.s, z31.s, z5.s[2]\n"
164
"fmla z20.s, z31.s, z1.s[2]\n"
165
"mov z1.d, z10.d\n"
166
"fmla z23.s, z31.s, z0.s[0]\n"
167
"fmla z19.s, z31.s, z0.s[2]\n"
168
"mov z0.d, z11.d\n"
169
"ld1w { z29.s }, p1/Z, [%x[params], #5, MUL VL]\n"
170
"fmla z24.s, z18.s, z6.s[0]\n"
171
"fmla z27.s, z18.s, z1.s[0]\n"
172
"fmla z26.s, z18.s, z1.s[2]\n"
173
"fmla z28.s, z18.s, z0.s[0]\n"
174
"mov z1.d, z14.d\n"
175
"mov z0.d, z15.d\n"
176
"fmla z21.s, z18.s, z6.s[2]\n"
177
"fmla z25.s, z18.s, z7.s[0]\n"
178
"fmla z20.s, z18.s, z1.s[0]\n"
179
"fmla z23.s, z18.s, z1.s[2]\n"
180
"mov z1.d, z10.d\n"
181
"fmla z19.s, z18.s, z0.s[0]\n"
182
"mov z0.d, z11.d\n"
183
"fmla z24.s, z17.s, z6.s[1]\n"
184
"ld1w { z31.s }, p2/Z, [%x[params], #7, MUL VL]\n"
185
"fmla z27.s, z17.s, z1.s[1]\n"
186
"fmla z26.s, z17.s, z1.s[3]\n"
187
"mov z1.d, z14.d\n"
188
"fmla z28.s, z17.s, z0.s[1]\n"
189
"mov z0.d, z15.d\n"
190
"fmla z21.s, z17.s, z6.s[3]\n"
191
"fmla z25.s, z17.s, z7.s[1]\n"
192
"fmla z20.s, z17.s, z1.s[1]\n"
193
"fmla z23.s, z17.s, z1.s[3]\n"
194
"fmla z19.s, z17.s, z0.s[1]\n"
195
"mov z1.d, z10.d\n"
196
"mov z0.d, z11.d\n"
197
"fmla z24.s, z29.s, z6.s[2]\n"
198
"fmla z27.s, z29.s, z1.s[2]\n"
199
"fmin z24.s, p1/M, z24.s, z16.s\n"
200
"fmla z26.s, z29.s, z0.s[0]\n"
201
"fmla z28.s, z29.s, z0.s[2]\n"
202
"mov z1.d, z14.d\n"
203
"fmax z24.s, p1/M, z24.s, z22.s\n"
204
"mov z0.d, z15.d\n"
205
"fmla z21.s, z29.s, z7.s[0]\n"
206
"fmla z25.s, z29.s, z7.s[2]\n"
207
"fmin z21.s, p1/M, z21.s, z16.s\n"
208
"fmla z20.s, z29.s, z1.s[2]\n"
209
"fmla z23.s, z29.s, z0.s[0]\n"
210
"fmin z25.s, p1/M, z25.s, z16.s\n"
211
"fmin z27.s, p1/M, z27.s, z16.s\n"
212
"fmla z19.s, z29.s, z0.s[2]\n"
213
"fmin z26.s, p1/M, z26.s, z16.s\n"
214
"fmin z28.s, p1/M, z28.s, z16.s\n"
215
"st1w { z24.s }, p0, [x28, x12, LSL #2]\n"
216
"fmin z20.s, p1/M, z20.s, z16.s\n"
217
"fmin z23.s, p1/M, z23.s, z16.s\n"
218
"ld1w { z24.s }, p2/Z, [%x[params], #6, MUL VL]\n"
219
"fmin z19.s, p1/M, z19.s, z16.s\n"
220
"addvl %x[params], %x[params], #16\n"
221
"ld1w { z30.s }, p2/Z, [%x[params], #-8, MUL VL]\n"
222
"ld1w { z29.s }, p2/Z, [%x[params], #-7, MUL VL]\n"
223
"fmax z21.s, p1/M, z21.s, z22.s\n"
224
"fmax z25.s, p1/M, z25.s, z22.s\n"
225
"st1w { z21.s }, p0, [x27, x12, LSL #2]\n"
226
"mov z21.d, z24.d\n"
227
"fmax z27.s, p1/M, z27.s, z22.s\n"
228
"fmax z26.s, p1/M, z26.s, z22.s\n"
229
"st1w { z25.s }, p0, [x26, x12, LSL #2]\n"
230
"mov z25.d, z24.d\n"
231
"fmax z28.s, p1/M, z28.s, z22.s\n"
232
"fmax z20.s, p1/M, z20.s, z22.s\n"
233
"st1w { z27.s }, p0, [x25, x12, LSL #2]\n"
234
"mov z27.d, z24.d\n"
235
"fmax z23.s, p1/M, z23.s, z22.s\n"
236
"fmax z19.s, p1/M, z19.s, z22.s\n"
237
"st1w { z26.s }, p0, [x24, x12, LSL #2]\n"
238
"mov z26.d, z24.d\n"
239
"st1w { z28.s }, p0, [x23, x12, LSL #2]\n"
240
"mov z28.d, z24.d\n"
241
"addvl %x[params], %x[params], #-6\n"
242
"st1w { z20.s }, p0, [x22, x12, LSL #2]\n"
243
"mov z20.d, z24.d\n"
244
"st1w { z23.s }, p0, [x21, x12, LSL #2]\n"
245
"mov z23.d, z24.d\n"
246
"st1w { z19.s }, p0, [x20, x12, LSL #2]\n"
247
"incw x12\n"
248
"mov z19.d, z24.d\n"
249
"b.any 1b\n"
250
: [params]
"+&r"
(params)
251
: [channel_multiplier]
"r"
(n_output_channels), [clamps]
"r"
(minmax_vals), [inptrs]
"r"
(inptrs), [outptrs]
"r"
(outptrs)
252
:
"cc"
,
"memory"
,
"p0"
,
"p1"
,
"p2"
,
"x9"
,
"x10"
,
"x11"
,
"x12"
,
"x13"
,
"x14"
,
"x15"
,
"x16"
,
"x17"
,
"x20"
,
"x21"
,
"x22"
,
"x23"
,
"x24"
,
"x25"
,
"x26"
,
"x27"
,
"x28"
,
"z0"
,
"z1"
,
"z2"
,
"z3"
,
"z4"
,
"z5"
,
"z6"
,
"z7"
,
"z8"
,
"z9"
,
"z10"
,
"z11"
,
"z12"
,
"z13"
,
"z14"
,
"z15"
,
"z16"
,
"z17"
,
"z18"
,
"z19"
,
"z20"
,
"z21"
,
"z22"
,
"z23"
,
"z24"
,
"z25"
,
"z26"
,
"z27"
,
"z28"
,
"z29"
,
"z30"
,
"z31"
253
);
254
}
255
256
}
// namespace depthwise
257
}
// namespace arm_conv
258
259
#endif // defined(ARM_COMPUTE_ENABLE_SVE)
arm_conv::depthwise::depthwise
template UniqueDepthwiseCommon< float > depthwise(const DepthwiseArgs &, const Nothing &)
activation_min
T activation_min
Definition:
working_space.hpp:145
activation_max
T activation_max
Definition:
working_space.hpp:145
arm_conv
Definition:
addressing.cpp:30
src
core
NEON
kernels
arm_conv
depthwise
kernels
sve_fp32_packed_to_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst
generic.cpp
Generated on Mon Mar 18 2024 11:31:57 for Compute Library by
1.8.17