30 #if defined(__aarch64__)
35 void a64_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst_impl(
36 const unsigned int n_channels,
37 const uint8_t *
const *
const inptrs,
38 const int8_t *
const weights,
39 const int32_t *
const bias,
43 uint8_t *
const *
const outptrs
48 long unsigned int n_channels;
54 uint8_t *
const *
const outptrs;
55 const uint8_t *inptrs[36];
58 long unsigned int n_channels,
59 const uint8_t *
const *inptrs_raw,
60 const void *
const weights,
61 const int32_t *
const bias,
65 uint8_t *
const *outptrs
66 ) : n_channels(n_channels), weights(weights),
bias(
bias),
70 inptrs[0] = inptrs_raw[0];
71 inptrs[1] = inptrs_raw[1];
72 inptrs[2] = inptrs_raw[6];
73 inptrs[3] = inptrs_raw[7];
74 inptrs[4] = inptrs_raw[2];
75 inptrs[5] = inptrs_raw[8];
76 inptrs[6] = inptrs_raw[3];
77 inptrs[7] = inptrs_raw[4];
78 inptrs[8] = inptrs_raw[11];
79 inptrs[9] = inptrs_raw[12];
80 inptrs[10] = inptrs_raw[9];
81 inptrs[11] = inptrs_raw[10];
82 inptrs[12] = inptrs_raw[5];
83 inptrs[13] = inptrs_raw[13];
84 inptrs[14] = inptrs_raw[14];
85 inptrs[15] = inptrs_raw[15];
86 inptrs[16] = inptrs_raw[16];
87 inptrs[17] = inptrs_raw[17];
88 inptrs[18] = inptrs_raw[18];
89 inptrs[19] = inptrs_raw[19];
90 inptrs[20] = inptrs_raw[20];
91 inptrs[21] = inptrs_raw[21];
92 inptrs[22] = inptrs_raw[22];
93 inptrs[23] = inptrs_raw[23];
94 inptrs[24] = inptrs_raw[24];
95 inptrs[25] = inptrs_raw[25];
96 inptrs[26] = inptrs_raw[26];
97 inptrs[27] = inptrs_raw[27];
98 inptrs[28] = inptrs_raw[28];
99 inptrs[29] = inptrs_raw[29];
100 inptrs[30] = inptrs_raw[30];
101 inptrs[31] = inptrs_raw[31];
102 inptrs[32] = inptrs_raw[32];
103 inptrs[33] = inptrs_raw[33];
104 inptrs[34] = inptrs_raw[34];
105 inptrs[35] = inptrs_raw[35];
110 const Params params(n_channels, inptrs, weights,
bias, qp,
113 __asm__ __volatile__(
114 "ldr x1, [%x[params], %[offsetof_Params_n_channels]]\n"
115 "ldr x23, [%x[params], %[offsetof_Params_requant]]\n"
117 "add x20, x23, %[offsetof_Requantize32_a_offset]\n"
118 "ld1r { v18.16b }, [x20]\n"
119 "ldr x22, [%x[params], %[offsetof_Params_outptrs]]\n"
120 "add x21, x23, %[offsetof_Requantize32_b_offset]\n"
121 "add x20, x23, %[offsetof_Requantize32_c_offset]\n"
122 "ld1r { v13.16b }, [x21]\n"
123 "ld1r { v26.8h }, [x20]\n"
124 "add x21, x23, %[offsetof_Requantize32_minval]\n"
125 "add x20, x23, %[offsetof_Requantize32_maxval]\n"
126 "ld1r { v11.8h }, [x21]\n"
127 "ld1r { v0.8h }, [x20]\n"
130 "add x5, %x[params], %[offsetof_Params_inptrs]\n"
131 "ldr x6, [%x[params], %[offsetof_Params_weights]]\n"
132 "ldr x7, [%x[params], %[offsetof_Params_requant_muls]]\n"
133 "ldr x8, [%x[params], %[offsetof_Params_requant_shifts]]\n"
134 "ldp x17, x16, [x22, #0x0]\n"
135 "ldp x15, x14, [x22, #0x10]\n"
137 "ldr d6, [x6, #0x0]\n"
138 "ldr d14, [x6, #0x8]\n"
139 "subs x2, x2, #0x1\n"
140 "ssubl v6.8h, v6.8b, v13.8b\n"
141 "ldr d10, [x6, #0x10]\n"
142 "ldr d21, [x6, #0x18]\n"
143 "ssubl v14.8h, v14.8b, v13.8b\n"
144 "ssubl v10.8h, v10.8b, v13.8b\n"
145 "ldr d12, [x6, #0x20]\n"
146 "ldr x20, [%x[params], %[offsetof_Params_bias]]\n"
147 "ssubl v21.8h, v21.8b, v13.8b\n"
148 "ssubl v12.8h, v12.8b, v13.8b\n"
149 "ldr q7, [x20, #0x0]\n"
150 "ldr q15, [x20, #0x10]\n"
151 "add x20, x20, #0x20\n"
152 "str x20, [%x[params], %[offsetof_Params_bias]]\n"
153 "ldp x9, x28, [x5, #0x0]\n"
154 "ldp x27, x26, [x5, #0x10]\n"
155 "mov v20.16b, v7.16b\n"
156 "mov v5.16b, v15.16b\n"
157 "ldp x25, x24, [x5, #0x20]\n"
158 "ldp x23, x22, [x5, #0x30]\n"
159 "mov v24.16b, v7.16b\n"
160 "mov v22.16b, v15.16b\n"
161 "ldp x21, x20, [x5, #0x40]\n"
162 "ldr d31, [x9, x3]\n"
163 "mov v23.16b, v7.16b\n"
164 "mov v19.16b, v15.16b\n"
165 "ldr d17, [x28, x3]\n"
166 "ldr d30, [x27, x3]\n"
167 "usubl v31.8h, v31.8b, v18.8b\n"
168 "usubl v17.8h, v17.8b, v18.8b\n"
169 "ldr d16, [x26, x3]\n"
170 "ldr d3, [x25, x3]\n"
171 "usubl v30.8h, v30.8b, v18.8b\n"
172 "usubl v16.8h, v16.8b, v18.8b\n"
173 "ldr d4, [x24, x3]\n"
174 "ldr d25, [x23, x3]\n"
175 "usubl v3.8h, v3.8b, v18.8b\n"
176 "usubl v4.8h, v4.8b, v18.8b\n"
177 "ldr d9, [x22, x3]\n"
178 "ldr d29, [x21, x3]\n"
179 "usubl v25.8h, v25.8b, v18.8b\n"
180 "usubl v9.8h, v9.8b, v18.8b\n"
181 "ldr d28, [x20, x3]\n"
182 "usubl v29.8h, v29.8b, v18.8b\n"
183 "usubl v28.8h, v28.8b, v18.8b\n"
186 "ldr d2, [x6, #0x28]\n"
187 "ldr d27, [x6, #0x30]\n"
188 "smlal v7.4s, v31.4h, v6.4h\n"
189 "smlal2 v15.4s, v31.8h, v6.8h\n"
190 "ldr d1, [x6, #0x38]\n"
191 "ldr d31, [x6, #0x40]\n"
192 "smlal v7.4s, v17.4h, v14.4h\n"
193 "smlal v20.4s, v17.4h, v6.4h\n"
194 "ldr d8, [x6, #0x48]\n"
195 "ldr x22, [x5, #0x50]\n"
196 "smlal v24.4s, v30.4h, v6.4h\n"
197 "smlal v23.4s, v16.4h, v6.4h\n"
198 "smlal2 v15.4s, v17.8h, v14.8h\n"
199 "smlal v7.4s, v3.4h, v10.4h\n"
200 "ldr x20, [x5, #0x58]\n"
201 "ldr x21, [x5, #0x60]\n"
202 "smlal2 v5.4s, v17.8h, v6.8h\n"
203 "ldr d17, [x22, x3]\n"
204 "smlal2 v22.4s, v30.8h, v6.8h\n"
205 "usubl v17.8h, v17.8b, v18.8b\n"
206 "smlal2 v19.4s, v16.8h, v6.8h\n"
207 "ldr d6, [x20, x3]\n"
208 "smlal v20.4s, v3.4h, v14.4h\n"
209 "usubl v6.8h, v6.8b, v18.8b\n"
210 "smlal v24.4s, v16.4h, v14.4h\n"
211 "smlal v23.4s, v4.4h, v14.4h\n"
212 "ssubl v2.8h, v2.8b, v13.8b\n"
213 "ldr x20, [x5, #0x68]\n"
214 "smlal2 v15.4s, v3.8h, v10.8h\n"
215 "smlal v7.4s, v25.4h, v21.4h\n"
216 "ssubl v27.8h, v27.8b, v13.8b\n"
217 "ldr x22, [x5, #0x70]\n"
218 "smlal2 v5.4s, v3.8h, v14.8h\n"
219 "ldr d3, [x21, x3]\n"
220 "smlal2 v22.4s, v16.8h, v14.8h\n"
221 "usubl v3.8h, v3.8b, v18.8b\n"
222 "smlal2 v19.4s, v4.8h, v14.8h\n"
223 "ldr d14, [x20, x3]\n"
224 "smlal v20.4s, v25.4h, v10.4h\n"
225 "usubl v14.8h, v14.8b, v18.8b\n"
226 "smlal v24.4s, v4.4h, v10.4h\n"
227 "smlal v23.4s, v17.4h, v10.4h\n"
228 "ssubl v1.8h, v1.8b, v13.8b\n"
229 "ldr x20, [x5, #0x78]\n"
230 "smlal2 v15.4s, v25.8h, v21.8h\n"
231 "smlal v7.4s, v9.4h, v12.4h\n"
232 "ssubl v31.8h, v31.8b, v13.8b\n"
233 "ldr x21, [x5, #0x80]\n"
234 "smlal2 v5.4s, v25.8h, v10.8h\n"
235 "ldr d25, [x22, x3]\n"
236 "smlal2 v22.4s, v4.8h, v10.8h\n"
237 "usubl v25.8h, v25.8b, v18.8b\n"
238 "smlal2 v19.4s, v17.8h, v10.8h\n"
239 "ldr d10, [x20, x3]\n"
240 "smlal v20.4s, v9.4h, v21.4h\n"
241 "usubl v10.8h, v10.8b, v18.8b\n"
242 "smlal v24.4s, v17.4h, v21.4h\n"
243 "smlal v23.4s, v6.4h, v21.4h\n"
244 "ssubl v8.8h, v8.8b, v13.8b\n"
245 "ldr x24, [x5, #0x88]\n"
246 "smlal2 v15.4s, v9.8h, v12.8h\n"
247 "smlal v7.4s, v30.4h, v2.4h\n"
248 "ldr x20, [x5, #0x90]\n"
249 "ldr x23, [x5, #0x98]\n"
250 "smlal2 v5.4s, v9.8h, v21.8h\n"
251 "ldr d9, [x21, x3]\n"
252 "smlal2 v22.4s, v17.8h, v21.8h\n"
253 "usubl v9.8h, v9.8b, v18.8b\n"
254 "smlal2 v19.4s, v6.8h, v21.8h\n"
255 "ldr d21, [x6, #0x50]\n"
256 "smlal v20.4s, v3.4h, v12.4h\n"
257 "ssubl v21.8h, v21.8b, v13.8b\n"
258 "smlal v24.4s, v6.4h, v12.4h\n"
259 "smlal v23.4s, v29.4h, v12.4h\n"
260 "ldr x22, [x5, #0xa0]\n"
261 "ldr x21, [x5, #0xa8]\n"
262 "smlal2 v15.4s, v30.8h, v2.8h\n"
263 "ldr d30, [x24, x3]\n"
264 "smlal v7.4s, v16.4h, v27.4h\n"
265 "usubl v30.8h, v30.8b, v18.8b\n"
266 "smlal2 v5.4s, v3.8h, v12.8h\n"
267 "ldr d3, [x6, #0x58]\n"
268 "smlal2 v22.4s, v6.8h, v12.8h\n"
269 "ssubl v3.8h, v3.8b, v13.8b\n"
270 "smlal2 v19.4s, v29.8h, v12.8h\n"
271 "ldr d12, [x20, x3]\n"
272 "smlal v20.4s, v16.4h, v2.4h\n"
273 "usubl v12.8h, v12.8b, v18.8b\n"
274 "smlal v24.4s, v28.4h, v2.4h\n"
275 "smlal v23.4s, v14.4h, v2.4h\n"
276 "ldr x20, [x5, #0xb0]\n"
277 "ldr x13, [x5, #0xb8]\n"
278 "smlal2 v15.4s, v16.8h, v27.8h\n"
279 "smlal v7.4s, v4.4h, v1.4h\n"
280 "ldr x12, [x5, #0xc0]\n"
281 "ldr x11, [x5, #0xc8]\n"
282 "smlal2 v5.4s, v16.8h, v2.8h\n"
283 "ldr d16, [x23, x3]\n"
284 "smlal2 v22.4s, v28.8h, v2.8h\n"
285 "usubl v16.8h, v16.8b, v18.8b\n"
286 "smlal2 v19.4s, v14.8h, v2.8h\n"
287 "ldr d2, [x6, #0x60]\n"
288 "smlal v20.4s, v4.4h, v27.4h\n"
289 "ssubl v2.8h, v2.8b, v13.8b\n"
290 "smlal v24.4s, v14.4h, v27.4h\n"
291 "smlal v23.4s, v25.4h, v27.4h\n"
292 "ldr x10, [x5, #0xd0]\n"
293 "ldr x9, [x5, #0xd8]\n"
294 "smlal2 v15.4s, v4.8h, v1.8h\n"
295 "smlal v7.4s, v17.4h, v31.4h\n"
296 "ldr x28, [x5, #0xe0]\n"
297 "ldr x27, [x5, #0xe8]\n"
298 "smlal2 v5.4s, v4.8h, v27.8h\n"
299 "ldr d4, [x22, x3]\n"
300 "smlal2 v22.4s, v14.8h, v27.8h\n"
301 "usubl v4.8h, v4.8b, v18.8b\n"
302 "smlal2 v19.4s, v25.8h, v27.8h\n"
303 "ldr d27, [x6, #0x68]\n"
304 "smlal v20.4s, v17.4h, v1.4h\n"
305 "ssubl v27.8h, v27.8b, v13.8b\n"
306 "smlal v24.4s, v25.4h, v1.4h\n"
307 "smlal v23.4s, v10.4h, v1.4h\n"
308 "ldr x26, [x5, #0xf0]\n"
309 "ldr x25, [x5, #0xf8]\n"
310 "smlal2 v15.4s, v17.8h, v31.8h\n"
311 "smlal v7.4s, v6.4h, v8.4h\n"
312 "ldr x24, [x5, #0x100]\n"
313 "ldr x23, [x5, #0x108]\n"
314 "smlal2 v5.4s, v17.8h, v1.8h\n"
315 "ldr d17, [x21, x3]\n"
316 "smlal2 v22.4s, v25.8h, v1.8h\n"
317 "usubl v17.8h, v17.8b, v18.8b\n"
318 "smlal2 v19.4s, v10.8h, v1.8h\n"
319 "ldr d1, [x6, #0x70]\n"
320 "smlal v20.4s, v6.4h, v31.4h\n"
321 "ssubl v1.8h, v1.8b, v13.8b\n"
322 "smlal v24.4s, v10.4h, v31.4h\n"
323 "smlal v23.4s, v9.4h, v31.4h\n"
324 "ldr x22, [x5, #0x110]\n"
325 "ldr x21, [x5, #0x118]\n"
326 "smlal2 v15.4s, v6.8h, v8.8h\n"
327 "smlal v7.4s, v28.4h, v21.4h\n"
328 "subs x2, x2, #0x1\n"
329 "smlal2 v5.4s, v6.8h, v31.8h\n"
330 "ldr d6, [x20, x3]\n"
331 "smlal2 v22.4s, v10.8h, v31.8h\n"
332 "usubl v6.8h, v6.8b, v18.8b\n"
333 "smlal2 v19.4s, v9.8h, v31.8h\n"
334 "ldr d31, [x6, #0x78]\n"
335 "smlal v20.4s, v29.4h, v8.4h\n"
336 "ssubl v31.8h, v31.8b, v13.8b\n"
337 "smlal v24.4s, v9.4h, v8.4h\n"
338 "smlal v23.4s, v30.4h, v8.4h\n"
339 "ldr x20, [%x[params], %[offsetof_Params_bias]]\n"
340 "smlal2 v15.4s, v28.8h, v21.8h\n"
341 "ldr d28, [x13, x3]\n"
342 "smlal v7.4s, v14.4h, v3.4h\n"
343 "usubl v28.8h, v28.8b, v18.8b\n"
344 "smlal2 v5.4s, v29.8h, v8.8h\n"
345 "ldr d29, [x6, #0x80]\n"
346 "smlal2 v22.4s, v9.8h, v8.8h\n"
347 "ssubl v29.8h, v29.8b, v13.8b\n"
348 "smlal2 v19.4s, v30.8h, v8.8h\n"
349 "ldr d8, [x12, x3]\n"
350 "smlal v20.4s, v14.4h, v21.4h\n"
351 "usubl v8.8h, v8.8b, v18.8b\n"
352 "smlal v24.4s, v12.4h, v21.4h\n"
353 "smlal v23.4s, v16.4h, v21.4h\n"
354 "smlal2 v15.4s, v14.8h, v3.8h\n"
355 "smlal v7.4s, v25.4h, v2.4h\n"
356 "smlal2 v5.4s, v14.8h, v21.8h\n"
357 "ldr d14, [x11, x3]\n"
358 "smlal2 v22.4s, v12.8h, v21.8h\n"
359 "usubl v14.8h, v14.8b, v18.8b\n"
360 "smlal2 v19.4s, v16.8h, v21.8h\n"
361 "ldr d21, [x6, #0x88]\n"
362 "smlal v20.4s, v25.4h, v3.4h\n"
363 "ssubl v21.8h, v21.8b, v13.8b\n"
364 "smlal v24.4s, v16.4h, v3.4h\n"
365 "smlal v23.4s, v4.4h, v3.4h\n"
366 "smlal2 v15.4s, v25.8h, v2.8h\n"
367 "smlal v7.4s, v10.4h, v27.4h\n"
368 "smlal2 v5.4s, v25.8h, v3.8h\n"
369 "ldr d25, [x10, x3]\n"
370 "smlal2 v22.4s, v16.8h, v3.8h\n"
371 "usubl v25.8h, v25.8b, v18.8b\n"
372 "smlal2 v19.4s, v4.8h, v3.8h\n"
373 "ldr d3, [x6, #0x90]\n"
374 "smlal v20.4s, v10.4h, v2.4h\n"
375 "ssubl v3.8h, v3.8b, v13.8b\n"
376 "smlal v24.4s, v4.4h, v2.4h\n"
377 "smlal v23.4s, v17.4h, v2.4h\n"
378 "smlal2 v15.4s, v10.8h, v27.8h\n"
379 "smlal v7.4s, v9.4h, v1.4h\n"
380 "smlal2 v5.4s, v10.8h, v2.8h\n"
381 "ldr d10, [x9, x3]\n"
382 "smlal2 v22.4s, v4.8h, v2.8h\n"
383 "usubl v10.8h, v10.8b, v18.8b\n"
384 "smlal2 v19.4s, v17.8h, v2.8h\n"
385 "ldr d2, [x6, #0x98]\n"
386 "smlal v20.4s, v9.4h, v27.4h\n"
387 "ssubl v2.8h, v2.8b, v13.8b\n"
388 "smlal v24.4s, v17.4h, v27.4h\n"
389 "smlal v23.4s, v6.4h, v27.4h\n"
390 "smlal2 v15.4s, v9.8h, v1.8h\n"
391 "smlal v7.4s, v12.4h, v31.4h\n"
392 "smlal2 v5.4s, v9.8h, v27.8h\n"
393 "ldr d9, [x28, x3]\n"
394 "smlal2 v22.4s, v17.8h, v27.8h\n"
395 "usubl v9.8h, v9.8b, v18.8b\n"
396 "smlal2 v19.4s, v6.8h, v27.8h\n"
397 "ldr d27, [x6, #0xa0]\n"
398 "smlal v20.4s, v30.4h, v1.4h\n"
399 "ssubl v27.8h, v27.8b, v13.8b\n"
400 "smlal v24.4s, v6.4h, v1.4h\n"
401 "smlal v23.4s, v28.4h, v1.4h\n"
402 "smlal2 v15.4s, v12.8h, v31.8h\n"
403 "ldr d12, [x27, x3]\n"
404 "smlal v7.4s, v16.4h, v29.4h\n"
405 "usubl v12.8h, v12.8b, v18.8b\n"
406 "smlal2 v5.4s, v30.8h, v1.8h\n"
407 "ldr d30, [x6, #0xa8]\n"
408 "smlal2 v22.4s, v6.8h, v1.8h\n"
409 "ssubl v30.8h, v30.8b, v13.8b\n"
410 "smlal2 v19.4s, v28.8h, v1.8h\n"
411 "ldr d1, [x26, x3]\n"
412 "smlal v20.4s, v16.4h, v31.4h\n"
413 "usubl v1.8h, v1.8b, v18.8b\n"
414 "smlal v24.4s, v8.4h, v31.4h\n"
415 "smlal v23.4s, v14.4h, v31.4h\n"
416 "smlal2 v15.4s, v16.8h, v29.8h\n"
417 "smlal v7.4s, v4.4h, v21.4h\n"
418 "smlal2 v5.4s, v16.8h, v31.8h\n"
419 "ldr d16, [x25, x3]\n"
420 "smlal2 v22.4s, v8.8h, v31.8h\n"
421 "usubl v16.8h, v16.8b, v18.8b\n"
422 "smlal2 v19.4s, v14.8h, v31.8h\n"
423 "ldr d31, [x6, #0xb0]\n"
424 "smlal v20.4s, v4.4h, v29.4h\n"
425 "ssubl v31.8h, v31.8b, v13.8b\n"
426 "smlal v24.4s, v14.4h, v29.4h\n"
427 "smlal v23.4s, v25.4h, v29.4h\n"
428 "smlal2 v15.4s, v4.8h, v21.8h\n"
429 "smlal v7.4s, v17.4h, v3.4h\n"
430 "smlal2 v5.4s, v4.8h, v29.8h\n"
431 "ldr d4, [x24, x3]\n"
432 "smlal2 v22.4s, v14.8h, v29.8h\n"
433 "usubl v4.8h, v4.8b, v18.8b\n"
434 "smlal2 v19.4s, v25.8h, v29.8h\n"
435 "ldr d29, [x6, #0xb8]\n"
436 "smlal v20.4s, v17.4h, v21.4h\n"
437 "ssubl v29.8h, v29.8b, v13.8b\n"
438 "smlal v24.4s, v25.4h, v21.4h\n"
439 "smlal v23.4s, v10.4h, v21.4h\n"
440 "smlal2 v15.4s, v17.8h, v3.8h\n"
441 "smlal v7.4s, v6.4h, v2.4h\n"
442 "smlal2 v5.4s, v17.8h, v21.8h\n"
443 "ldr d17, [x23, x3]\n"
444 "smlal2 v22.4s, v25.8h, v21.8h\n"
445 "usubl v17.8h, v17.8b, v18.8b\n"
446 "smlal2 v19.4s, v10.8h, v21.8h\n"
447 "ldr d21, [x6, #0xc0]\n"
448 "smlal v20.4s, v6.4h, v3.4h\n"
449 "ssubl v21.8h, v21.8b, v13.8b\n"
450 "smlal v24.4s, v10.4h, v3.4h\n"
451 "smlal v23.4s, v9.4h, v3.4h\n"
452 "add x6, x6, #0xc8\n"
453 "smlal2 v15.4s, v6.8h, v2.8h\n"
454 "smlal v7.4s, v8.4h, v27.4h\n"
455 "smlal2 v5.4s, v6.8h, v3.8h\n"
456 "ldr d6, [x22, x3]\n"
457 "smlal2 v22.4s, v10.8h, v3.8h\n"
458 "usubl v6.8h, v6.8b, v18.8b\n"
459 "smlal2 v19.4s, v9.8h, v3.8h\n"
460 "ldr d3, [x21, x3]\n"
461 "smlal v20.4s, v28.4h, v2.4h\n"
462 "usubl v3.8h, v3.8b, v18.8b\n"
463 "smlal v24.4s, v9.4h, v2.4h\n"
464 "smlal v23.4s, v12.4h, v2.4h\n"
466 "smlal2 v15.4s, v8.8h, v27.8h\n"
467 "ldr q8, [x7, #0x0]\n"
468 "smlal v7.4s, v14.4h, v30.4h\n"
469 "smlal2 v5.4s, v28.8h, v2.8h\n"
470 "ldr q28, [x8, #0x0]\n"
471 "smlal2 v22.4s, v9.8h, v2.8h\n"
472 "smlal2 v19.4s, v12.8h, v2.8h\n"
473 "ldr q2, [x7, #0x10]\n"
474 "smlal v20.4s, v14.4h, v27.4h\n"
475 "add x7, x7, #0x20\n"
476 "smlal v24.4s, v1.4h, v27.4h\n"
477 "smlal v23.4s, v16.4h, v27.4h\n"
478 "smlal2 v15.4s, v14.8h, v30.8h\n"
479 "smlal v7.4s, v25.4h, v31.4h\n"
480 "smlal2 v5.4s, v14.8h, v27.8h\n"
481 "ldr q14, [x8, #0x10]\n"
482 "smlal2 v22.4s, v1.8h, v27.8h\n"
483 "add x8, x8, #0x20\n"
484 "smlal2 v19.4s, v16.8h, v27.8h\n"
485 "smlal v20.4s, v25.4h, v30.4h\n"
486 "smlal v24.4s, v16.4h, v30.4h\n"
487 "smlal v23.4s, v4.4h, v30.4h\n"
488 "smlal2 v15.4s, v25.8h, v31.8h\n"
489 "smlal v7.4s, v10.4h, v29.4h\n"
490 "smlal2 v5.4s, v25.8h, v30.8h\n"
491 "smlal2 v22.4s, v16.8h, v30.8h\n"
492 "smlal2 v19.4s, v4.8h, v30.8h\n"
493 "smlal v20.4s, v10.4h, v31.4h\n"
494 "smlal v24.4s, v4.4h, v31.4h\n"
495 "smlal v23.4s, v17.4h, v31.4h\n"
496 "smlal2 v15.4s, v10.8h, v29.8h\n"
497 "smlal v7.4s, v9.4h, v21.4h\n"
498 "sqrdmulh v7.4s, v7.4s, v8.4s\n"
499 "smlal2 v5.4s, v10.8h, v31.8h\n"
500 "smlal2 v22.4s, v4.8h, v31.8h\n"
501 "and v27.16b, v7.16b, v28.16b\n"
502 "smlal2 v19.4s, v17.8h, v31.8h\n"
503 "smlal v20.4s, v9.4h, v29.4h\n"
504 "sshr v27.4s, v27.4s, #0x1f\n"
505 "smlal v24.4s, v17.4h, v29.4h\n"
506 "smlal v23.4s, v6.4h, v29.4h\n"
507 "sqadd v7.4s, v7.4s, v27.4s\n"
508 "smlal2 v15.4s, v9.8h, v21.8h\n"
509 "smlal2 v5.4s, v9.8h, v29.8h\n"
510 "sqrdmulh v15.4s, v15.4s, v2.4s\n"
511 "smlal2 v22.4s, v17.8h, v29.8h\n"
512 "smlal2 v19.4s, v6.8h, v29.8h\n"
513 "and v9.16b, v15.16b, v14.16b\n"
514 "smlal v20.4s, v12.4h, v21.4h\n"
515 "smlal v24.4s, v6.4h, v21.4h\n"
516 "sqrdmulh v20.4s, v20.4s, v8.4s\n"
517 "smlal v23.4s, v3.4h, v21.4h\n"
518 "smlal2 v5.4s, v12.8h, v21.8h\n"
519 "sqrdmulh v24.4s, v24.4s, v8.4s\n"
520 "smlal2 v22.4s, v6.8h, v21.8h\n"
521 "smlal2 v19.4s, v3.8h, v21.8h\n"
522 "sqrdmulh v23.4s, v23.4s, v8.4s\n"
523 "sshr v9.4s, v9.4s, #0x1f\n"
524 "and v25.16b, v20.16b, v28.16b\n"
525 "sqrdmulh v5.4s, v5.4s, v2.4s\n"
526 "and v10.16b, v24.16b, v28.16b\n"
527 "sqrdmulh v22.4s, v22.4s, v2.4s\n"
528 "and v21.16b, v23.16b, v28.16b\n"
529 "sqrdmulh v19.4s, v19.4s, v2.4s\n"
530 "sqadd v15.4s, v15.4s, v9.4s\n"
531 "sshr v25.4s, v25.4s, #0x1f\n"
532 "and v9.16b, v5.16b, v14.16b\n"
533 "sshr v10.4s, v10.4s, #0x1f\n"
534 "and v12.16b, v22.16b, v14.16b\n"
535 "sshr v21.4s, v21.4s, #0x1f\n"
536 "and v17.16b, v19.16b, v14.16b\n"
537 "sqadd v20.4s, v20.4s, v25.4s\n"
538 "sshr v9.4s, v9.4s, #0x1f\n"
539 "sqadd v24.4s, v24.4s, v10.4s\n"
540 "sshr v12.4s, v12.4s, #0x1f\n"
541 "sqadd v23.4s, v23.4s, v21.4s\n"
542 "sshr v17.4s, v17.4s, #0x1f\n"
543 "srshl v7.4s, v7.4s, v28.4s\n"
544 "srshl v20.4s, v20.4s, v28.4s\n"
545 "sqadd v5.4s, v5.4s, v9.4s\n"
546 "srshl v24.4s, v24.4s, v28.4s\n"
547 "sqadd v22.4s, v22.4s, v12.4s\n"
548 "srshl v23.4s, v23.4s, v28.4s\n"
549 "sqadd v19.4s, v19.4s, v17.4s\n"
550 "srshl v15.4s, v15.4s, v14.4s\n"
551 "sqxtn v7.4h, v7.4s\n"
552 "srshl v5.4s, v5.4s, v14.4s\n"
553 "sqxtn v20.4h, v20.4s\n"
554 "srshl v22.4s, v22.4s, v14.4s\n"
555 "sqxtn v24.4h, v24.4s\n"
556 "srshl v19.4s, v19.4s, v14.4s\n"
557 "sqxtn v23.4h, v23.4s\n"
558 "sqxtn2 v7.8h, v15.4s\n"
559 "sqxtn2 v20.8h, v5.4s\n"
560 "sqxtn2 v24.8h, v22.4s\n"
561 "sqxtn2 v23.8h, v19.4s\n"
562 "sqadd v7.8h, v7.8h, v26.8h\n"
563 "sqadd v20.8h, v20.8h, v26.8h\n"
564 "sqadd v24.8h, v24.8h, v26.8h\n"
565 "sqadd v23.8h, v23.8h, v26.8h\n"
566 "smax v7.8h, v7.8h, v11.8h\n"
567 "smax v20.8h, v20.8h, v11.8h\n"
568 "smax v24.8h, v24.8h, v11.8h\n"
569 "smax v23.8h, v23.8h, v11.8h\n"
570 "smin v7.8h, v7.8h, v0.8h\n"
571 "smin v20.8h, v20.8h, v0.8h\n"
572 "smin v24.8h, v24.8h, v0.8h\n"
573 "smin v23.8h, v23.8h, v0.8h\n"
574 "uzp1 v7.16b, v7.16b, v7.16b\n"
575 "str d7, [x17, x4]\n"
576 "uzp1 v20.16b, v20.16b, v20.16b\n"
577 "uzp1 v24.16b, v24.16b, v24.16b\n"
578 "str d20, [x16, x4]\n"
579 "uzp1 v23.16b, v23.16b, v23.16b\n"
580 "str d24, [x15, x4]\n"
581 "str d23, [x14, x4]\n"
582 "ldr q7, [x20, #0x0]\n"
583 "ldr q15, [x20, #0x10]\n"
584 "add x20, x20, #0x20\n"
585 "ldr d6, [x6, #0x0]\n"
586 "ldr d14, [x6, #0x8]\n"
588 "str x20, [%x[params], %[offsetof_Params_bias]]\n"
589 "ldr d10, [x6, #0x10]\n"
590 "ldr d21, [x6, #0x18]\n"
591 "mov v20.16b, v7.16b\n"
592 "mov v5.16b, v15.16b\n"
593 "ldr d12, [x6, #0x20]\n"
594 "ldp x9, x28, [x5, #0x0]\n"
595 "mov v24.16b, v7.16b\n"
596 "mov v22.16b, v15.16b\n"
597 "ldp x27, x26, [x5, #0x10]\n"
598 "ldp x25, x24, [x5, #0x20]\n"
599 "mov v23.16b, v7.16b\n"
600 "mov v19.16b, v15.16b\n"
601 "ldp x23, x22, [x5, #0x30]\n"
602 "ldp x21, x20, [x5, #0x40]\n"
603 "ssubl v6.8h, v6.8b, v13.8b\n"
604 "ssubl v14.8h, v14.8b, v13.8b\n"
605 "ldr d31, [x9, x3]\n"
606 "ldr d17, [x28, x3]\n"
607 "ssubl v10.8h, v10.8b, v13.8b\n"
608 "ssubl v21.8h, v21.8b, v13.8b\n"
609 "ldr d30, [x27, x3]\n"
610 "ldr d16, [x26, x3]\n"
611 "ssubl v12.8h, v12.8b, v13.8b\n"
612 "usubl v31.8h, v31.8b, v18.8b\n"
613 "ldr d3, [x25, x3]\n"
614 "ldr d4, [x24, x3]\n"
615 "usubl v17.8h, v17.8b, v18.8b\n"
616 "usubl v30.8h, v30.8b, v18.8b\n"
617 "ldr d25, [x23, x3]\n"
618 "ldr d9, [x22, x3]\n"
619 "usubl v16.8h, v16.8b, v18.8b\n"
620 "usubl v3.8h, v3.8b, v18.8b\n"
621 "ldr d29, [x21, x3]\n"
622 "ldr d28, [x20, x3]\n"
623 "usubl v4.8h, v4.8b, v18.8b\n"
624 "usubl v25.8h, v25.8b, v18.8b\n"
625 "usubl v9.8h, v9.8b, v18.8b\n"
626 "usubl v29.8h, v29.8b, v18.8b\n"
627 "usubl v28.8h, v28.8b, v18.8b\n"
630 "ldr d27, [x6, #0x28]\n"
631 "ldr d1, [x6, #0x30]\n"
632 "smlal v7.4s, v31.4h, v6.4h\n"
633 "smlal2 v15.4s, v31.8h, v6.8h\n"
634 "ldr d2, [x6, #0x38]\n"
635 "ldr d31, [x6, #0x40]\n"
636 "smlal v7.4s, v17.4h, v14.4h\n"
637 "smlal v20.4s, v17.4h, v6.4h\n"
638 "ldr d8, [x6, #0x48]\n"
639 "ldr x22, [x5, #0x50]\n"
640 "smlal v24.4s, v30.4h, v6.4h\n"
641 "smlal v23.4s, v16.4h, v6.4h\n"
642 "smlal2 v15.4s, v17.8h, v14.8h\n"
643 "smlal v7.4s, v3.4h, v10.4h\n"
644 "ldr x20, [x5, #0x58]\n"
645 "ldr x21, [x5, #0x60]\n"
646 "smlal2 v5.4s, v17.8h, v6.8h\n"
647 "ldr d17, [x22, x3]\n"
648 "smlal2 v22.4s, v30.8h, v6.8h\n"
649 "usubl v17.8h, v17.8b, v18.8b\n"
650 "smlal2 v19.4s, v16.8h, v6.8h\n"
651 "ldr d6, [x20, x3]\n"
652 "smlal v20.4s, v3.4h, v14.4h\n"
653 "usubl v6.8h, v6.8b, v18.8b\n"
654 "smlal v24.4s, v16.4h, v14.4h\n"
655 "smlal v23.4s, v4.4h, v14.4h\n"
656 "ssubl v27.8h, v27.8b, v13.8b\n"
657 "ldr x20, [x5, #0x68]\n"
658 "smlal2 v15.4s, v3.8h, v10.8h\n"
659 "smlal v7.4s, v25.4h, v21.4h\n"
660 "ssubl v1.8h, v1.8b, v13.8b\n"
661 "ldr x22, [x5, #0x70]\n"
662 "smlal2 v5.4s, v3.8h, v14.8h\n"
663 "ldr d3, [x21, x3]\n"
664 "smlal2 v22.4s, v16.8h, v14.8h\n"
665 "usubl v3.8h, v3.8b, v18.8b\n"
666 "smlal2 v19.4s, v4.8h, v14.8h\n"
667 "ldr d14, [x20, x3]\n"
668 "smlal v20.4s, v25.4h, v10.4h\n"
669 "usubl v14.8h, v14.8b, v18.8b\n"
670 "smlal v24.4s, v4.4h, v10.4h\n"
671 "smlal v23.4s, v17.4h, v10.4h\n"
672 "ssubl v2.8h, v2.8b, v13.8b\n"
673 "ldr x21, [x5, #0x78]\n"
674 "smlal2 v15.4s, v25.8h, v21.8h\n"
675 "smlal v7.4s, v9.4h, v12.4h\n"
676 "ssubl v31.8h, v31.8b, v13.8b\n"
677 "ldr x20, [x5, #0x80]\n"
678 "smlal2 v5.4s, v25.8h, v10.8h\n"
679 "ldr d25, [x22, x3]\n"
680 "smlal2 v22.4s, v4.8h, v10.8h\n"
681 "usubl v25.8h, v25.8b, v18.8b\n"
682 "smlal2 v19.4s, v17.8h, v10.8h\n"
683 "ldr d10, [x21, x3]\n"
684 "smlal v20.4s, v9.4h, v21.4h\n"
685 "usubl v10.8h, v10.8b, v18.8b\n"
686 "smlal v24.4s, v17.4h, v21.4h\n"
687 "smlal v23.4s, v6.4h, v21.4h\n"
688 "ssubl v8.8h, v8.8b, v13.8b\n"
689 "ldr x24, [x5, #0x88]\n"
690 "smlal2 v15.4s, v9.8h, v12.8h\n"
691 "smlal v7.4s, v30.4h, v27.4h\n"
692 "ldr x23, [x5, #0x90]\n"
693 "ldr x22, [x5, #0x98]\n"
694 "smlal2 v5.4s, v9.8h, v21.8h\n"
695 "ldr d9, [x20, x3]\n"
696 "smlal2 v22.4s, v17.8h, v21.8h\n"
697 "usubl v9.8h, v9.8b, v18.8b\n"
698 "smlal2 v19.4s, v6.8h, v21.8h\n"
699 "ldr d21, [x6, #0x50]\n"
700 "smlal v20.4s, v3.4h, v12.4h\n"
701 "ssubl v21.8h, v21.8b, v13.8b\n"
702 "smlal v24.4s, v6.4h, v12.4h\n"
703 "smlal v23.4s, v29.4h, v12.4h\n"
704 "ldr x21, [x5, #0xa0]\n"
705 "ldr x20, [x5, #0xa8]\n"
706 "smlal2 v15.4s, v30.8h, v27.8h\n"
707 "ldr d30, [x24, x3]\n"
708 "smlal v7.4s, v16.4h, v1.4h\n"
709 "usubl v30.8h, v30.8b, v18.8b\n"
710 "smlal2 v5.4s, v3.8h, v12.8h\n"
711 "ldr d3, [x6, #0x58]\n"
712 "smlal2 v22.4s, v6.8h, v12.8h\n"
713 "ssubl v3.8h, v3.8b, v13.8b\n"
714 "smlal2 v19.4s, v29.8h, v12.8h\n"
715 "ldr d12, [x23, x3]\n"
716 "smlal v20.4s, v16.4h, v27.4h\n"
717 "usubl v12.8h, v12.8b, v18.8b\n"
718 "smlal v24.4s, v28.4h, v27.4h\n"
719 "smlal v23.4s, v14.4h, v27.4h\n"
720 "ldr x13, [x5, #0xb0]\n"
721 "ldr x12, [x5, #0xb8]\n"
722 "smlal2 v15.4s, v16.8h, v1.8h\n"
723 "smlal v7.4s, v4.4h, v2.4h\n"
724 "ldr x11, [x5, #0xc0]\n"
725 "ldr x10, [x5, #0xc8]\n"
726 "smlal2 v5.4s, v16.8h, v27.8h\n"
727 "ldr d16, [x22, x3]\n"
728 "smlal2 v22.4s, v28.8h, v27.8h\n"
729 "usubl v16.8h, v16.8b, v18.8b\n"
730 "smlal2 v19.4s, v14.8h, v27.8h\n"
731 "ldr d27, [x6, #0x60]\n"
732 "smlal v20.4s, v4.4h, v1.4h\n"
733 "ssubl v27.8h, v27.8b, v13.8b\n"
734 "smlal v24.4s, v14.4h, v1.4h\n"
735 "smlal v23.4s, v25.4h, v1.4h\n"
736 "ldr x9, [x5, #0xd0]\n"
737 "ldr x28, [x5, #0xd8]\n"
738 "smlal2 v15.4s, v4.8h, v2.8h\n"
739 "smlal v7.4s, v17.4h, v31.4h\n"
740 "ldr x27, [x5, #0xe0]\n"
741 "ldr x26, [x5, #0xe8]\n"
742 "smlal2 v5.4s, v4.8h, v1.8h\n"
743 "ldr d4, [x21, x3]\n"
744 "smlal2 v22.4s, v14.8h, v1.8h\n"
745 "usubl v4.8h, v4.8b, v18.8b\n"
746 "smlal2 v19.4s, v25.8h, v1.8h\n"
747 "ldr d1, [x6, #0x68]\n"
748 "smlal v20.4s, v17.4h, v2.4h\n"
749 "ssubl v1.8h, v1.8b, v13.8b\n"
750 "smlal v24.4s, v25.4h, v2.4h\n"
751 "smlal v23.4s, v10.4h, v2.4h\n"
752 "ldr x25, [x5, #0xf0]\n"
753 "ldr x24, [x5, #0xf8]\n"
754 "smlal2 v15.4s, v17.8h, v31.8h\n"
755 "smlal v7.4s, v6.4h, v8.4h\n"
756 "ldr x23, [x5, #0x100]\n"
757 "ldr x22, [x5, #0x108]\n"
758 "smlal2 v5.4s, v17.8h, v2.8h\n"
759 "ldr d17, [x20, x3]\n"
760 "smlal2 v22.4s, v25.8h, v2.8h\n"
761 "usubl v17.8h, v17.8b, v18.8b\n"
762 "smlal2 v19.4s, v10.8h, v2.8h\n"
763 "ldr d2, [x6, #0x70]\n"
764 "smlal v20.4s, v6.4h, v31.4h\n"
765 "ssubl v2.8h, v2.8b, v13.8b\n"
766 "smlal v24.4s, v10.4h, v31.4h\n"
767 "smlal v23.4s, v9.4h, v31.4h\n"
768 "ldr x21, [x5, #0x110]\n"
769 "ldr x20, [x5, #0x118]\n"
770 "smlal2 v15.4s, v6.8h, v8.8h\n"
771 "smlal v7.4s, v28.4h, v21.4h\n"
773 "smlal2 v5.4s, v6.8h, v31.8h\n"
774 "ldr d6, [x13, x3]\n"
775 "smlal2 v22.4s, v10.8h, v31.8h\n"
776 "usubl v6.8h, v6.8b, v18.8b\n"
777 "smlal2 v19.4s, v9.8h, v31.8h\n"
778 "ldr d31, [x6, #0x78]\n"
779 "smlal v20.4s, v29.4h, v8.4h\n"
780 "ssubl v31.8h, v31.8b, v13.8b\n"
781 "smlal v24.4s, v9.4h, v8.4h\n"
782 "smlal v23.4s, v30.4h, v8.4h\n"
783 "smlal2 v15.4s, v28.8h, v21.8h\n"
784 "ldr d28, [x12, x3]\n"
785 "smlal v7.4s, v14.4h, v3.4h\n"
786 "usubl v28.8h, v28.8b, v18.8b\n"
787 "smlal2 v5.4s, v29.8h, v8.8h\n"
788 "ldr d29, [x6, #0x80]\n"
789 "smlal2 v22.4s, v9.8h, v8.8h\n"
790 "ssubl v29.8h, v29.8b, v13.8b\n"
791 "smlal2 v19.4s, v30.8h, v8.8h\n"
792 "ldr d8, [x11, x3]\n"
793 "smlal v20.4s, v14.4h, v21.4h\n"
794 "usubl v8.8h, v8.8b, v18.8b\n"
795 "smlal v24.4s, v12.4h, v21.4h\n"
796 "smlal v23.4s, v16.4h, v21.4h\n"
797 "smlal2 v15.4s, v14.8h, v3.8h\n"
798 "smlal v7.4s, v25.4h, v27.4h\n"
799 "smlal2 v5.4s, v14.8h, v21.8h\n"
800 "ldr d14, [x10, x3]\n"
801 "smlal2 v22.4s, v12.8h, v21.8h\n"
802 "usubl v14.8h, v14.8b, v18.8b\n"
803 "smlal2 v19.4s, v16.8h, v21.8h\n"
804 "ldr d21, [x6, #0x88]\n"
805 "smlal v20.4s, v25.4h, v3.4h\n"
806 "ssubl v21.8h, v21.8b, v13.8b\n"
807 "smlal v24.4s, v16.4h, v3.4h\n"
808 "smlal v23.4s, v4.4h, v3.4h\n"
809 "smlal2 v15.4s, v25.8h, v27.8h\n"
810 "smlal v7.4s, v10.4h, v1.4h\n"
811 "smlal2 v5.4s, v25.8h, v3.8h\n"
812 "ldr d25, [x9, x3]\n"
813 "smlal2 v22.4s, v16.8h, v3.8h\n"
814 "usubl v25.8h, v25.8b, v18.8b\n"
815 "smlal2 v19.4s, v4.8h, v3.8h\n"
816 "ldr d3, [x6, #0x90]\n"
817 "smlal v20.4s, v10.4h, v27.4h\n"
818 "ssubl v3.8h, v3.8b, v13.8b\n"
819 "smlal v24.4s, v4.4h, v27.4h\n"
820 "smlal v23.4s, v17.4h, v27.4h\n"
821 "smlal2 v15.4s, v10.8h, v1.8h\n"
822 "smlal v7.4s, v9.4h, v2.4h\n"
823 "smlal2 v5.4s, v10.8h, v27.8h\n"
824 "ldr d10, [x28, x3]\n"
825 "smlal2 v22.4s, v4.8h, v27.8h\n"
826 "usubl v10.8h, v10.8b, v18.8b\n"
827 "smlal2 v19.4s, v17.8h, v27.8h\n"
828 "ldr d27, [x6, #0x98]\n"
829 "smlal v20.4s, v9.4h, v1.4h\n"
830 "ssubl v27.8h, v27.8b, v13.8b\n"
831 "smlal v24.4s, v17.4h, v1.4h\n"
832 "smlal v23.4s, v6.4h, v1.4h\n"
833 "smlal2 v15.4s, v9.8h, v2.8h\n"
834 "smlal v7.4s, v12.4h, v31.4h\n"
835 "smlal2 v5.4s, v9.8h, v1.8h\n"
836 "ldr d9, [x27, x3]\n"
837 "smlal2 v22.4s, v17.8h, v1.8h\n"
838 "usubl v9.8h, v9.8b, v18.8b\n"
839 "smlal2 v19.4s, v6.8h, v1.8h\n"
840 "ldr d1, [x6, #0xa0]\n"
841 "smlal v20.4s, v30.4h, v2.4h\n"
842 "ssubl v1.8h, v1.8b, v13.8b\n"
843 "smlal v24.4s, v6.4h, v2.4h\n"
844 "smlal v23.4s, v28.4h, v2.4h\n"
845 "smlal2 v15.4s, v12.8h, v31.8h\n"
846 "ldr d12, [x26, x3]\n"
847 "smlal v7.4s, v16.4h, v29.4h\n"
848 "usubl v12.8h, v12.8b, v18.8b\n"
849 "smlal2 v5.4s, v30.8h, v2.8h\n"
850 "ldr d30, [x6, #0xa8]\n"
851 "smlal2 v22.4s, v6.8h, v2.8h\n"
852 "ssubl v30.8h, v30.8b, v13.8b\n"
853 "smlal2 v19.4s, v28.8h, v2.8h\n"
854 "ldr d2, [x25, x3]\n"
855 "smlal v20.4s, v16.4h, v31.4h\n"
856 "usubl v2.8h, v2.8b, v18.8b\n"
857 "smlal v24.4s, v8.4h, v31.4h\n"
858 "smlal v23.4s, v14.4h, v31.4h\n"
859 "smlal2 v15.4s, v16.8h, v29.8h\n"
860 "smlal v7.4s, v4.4h, v21.4h\n"
861 "smlal2 v5.4s, v16.8h, v31.8h\n"
862 "ldr d16, [x24, x3]\n"
863 "smlal2 v22.4s, v8.8h, v31.8h\n"
864 "usubl v16.8h, v16.8b, v18.8b\n"
865 "smlal2 v19.4s, v14.8h, v31.8h\n"
866 "ldr d31, [x6, #0xb0]\n"
867 "smlal v20.4s, v4.4h, v29.4h\n"
868 "ssubl v31.8h, v31.8b, v13.8b\n"
869 "smlal v24.4s, v14.4h, v29.4h\n"
870 "smlal v23.4s, v25.4h, v29.4h\n"
871 "smlal2 v15.4s, v4.8h, v21.8h\n"
872 "smlal v7.4s, v17.4h, v3.4h\n"
873 "smlal2 v5.4s, v4.8h, v29.8h\n"
874 "ldr d4, [x23, x3]\n"
875 "smlal2 v22.4s, v14.8h, v29.8h\n"
876 "usubl v4.8h, v4.8b, v18.8b\n"
877 "smlal2 v19.4s, v25.8h, v29.8h\n"
878 "ldr d29, [x6, #0xb8]\n"
879 "smlal v20.4s, v17.4h, v21.4h\n"
880 "ssubl v29.8h, v29.8b, v13.8b\n"
881 "smlal v24.4s, v25.4h, v21.4h\n"
882 "smlal v23.4s, v10.4h, v21.4h\n"
883 "smlal2 v15.4s, v17.8h, v3.8h\n"
884 "smlal v7.4s, v6.4h, v27.4h\n"
885 "smlal2 v5.4s, v17.8h, v21.8h\n"
886 "ldr d17, [x22, x3]\n"
887 "smlal2 v22.4s, v25.8h, v21.8h\n"
888 "usubl v17.8h, v17.8b, v18.8b\n"
889 "smlal2 v19.4s, v10.8h, v21.8h\n"
890 "ldr d21, [x6, #0xc0]\n"
891 "smlal v20.4s, v6.4h, v3.4h\n"
892 "ssubl v21.8h, v21.8b, v13.8b\n"
893 "smlal v24.4s, v10.4h, v3.4h\n"
894 "smlal v23.4s, v9.4h, v3.4h\n"
895 "smlal2 v15.4s, v6.8h, v27.8h\n"
896 "smlal v7.4s, v8.4h, v1.4h\n"
897 "smlal2 v5.4s, v6.8h, v3.8h\n"
898 "ldr d6, [x21, x3]\n"
899 "smlal2 v22.4s, v10.8h, v3.8h\n"
900 "usubl v6.8h, v6.8b, v18.8b\n"
901 "smlal2 v19.4s, v9.8h, v3.8h\n"
902 "ldr d3, [x20, x3]\n"
903 "smlal v20.4s, v28.4h, v27.4h\n"
904 "usubl v3.8h, v3.8b, v18.8b\n"
905 "smlal v24.4s, v9.4h, v27.4h\n"
906 "smlal v23.4s, v12.4h, v27.4h\n"
908 "smlal2 v15.4s, v8.8h, v1.8h\n"
909 "ldr q8, [x7, #0x0]\n"
910 "smlal v7.4s, v14.4h, v30.4h\n"
911 "smlal2 v5.4s, v28.8h, v27.8h\n"
912 "ldr q28, [x8, #0x0]\n"
913 "smlal2 v22.4s, v9.8h, v27.8h\n"
914 "smlal2 v19.4s, v12.8h, v27.8h\n"
915 "ldr q27, [x7, #0x10]\n"
916 "smlal v20.4s, v14.4h, v1.4h\n"
917 "add x7, x7, #0x20\n"
918 "smlal v24.4s, v2.4h, v1.4h\n"
919 "smlal v23.4s, v16.4h, v1.4h\n"
920 "smlal2 v15.4s, v14.8h, v30.8h\n"
921 "smlal v7.4s, v25.4h, v31.4h\n"
922 "smlal2 v5.4s, v14.8h, v1.8h\n"
923 "ldr q14, [x8, #0x10]\n"
924 "smlal2 v22.4s, v2.8h, v1.8h\n"
925 "add x8, x8, #0x20\n"
926 "smlal2 v19.4s, v16.8h, v1.8h\n"
927 "smlal v20.4s, v25.4h, v30.4h\n"
928 "smlal v24.4s, v16.4h, v30.4h\n"
929 "smlal v23.4s, v4.4h, v30.4h\n"
930 "smlal2 v15.4s, v25.8h, v31.8h\n"
931 "smlal v7.4s, v10.4h, v29.4h\n"
932 "smlal2 v5.4s, v25.8h, v30.8h\n"
933 "smlal2 v22.4s, v16.8h, v30.8h\n"
934 "smlal2 v19.4s, v4.8h, v30.8h\n"
935 "smlal v20.4s, v10.4h, v31.4h\n"
936 "smlal v24.4s, v4.4h, v31.4h\n"
937 "smlal v23.4s, v17.4h, v31.4h\n"
938 "smlal2 v15.4s, v10.8h, v29.8h\n"
939 "smlal v7.4s, v9.4h, v21.4h\n"
940 "sqrdmulh v7.4s, v7.4s, v8.4s\n"
941 "smlal2 v5.4s, v10.8h, v31.8h\n"
942 "smlal2 v22.4s, v4.8h, v31.8h\n"
943 "and v4.16b, v7.16b, v28.16b\n"
944 "smlal2 v19.4s, v17.8h, v31.8h\n"
945 "smlal v20.4s, v9.4h, v29.4h\n"
946 "sshr v4.4s, v4.4s, #0x1f\n"
947 "smlal v24.4s, v17.4h, v29.4h\n"
948 "smlal v23.4s, v6.4h, v29.4h\n"
949 "sqadd v7.4s, v7.4s, v4.4s\n"
950 "smlal2 v15.4s, v9.8h, v21.8h\n"
951 "smlal2 v5.4s, v9.8h, v29.8h\n"
952 "sqrdmulh v15.4s, v15.4s, v27.4s\n"
953 "smlal2 v22.4s, v17.8h, v29.8h\n"
954 "smlal2 v19.4s, v6.8h, v29.8h\n"
955 "and v30.16b, v15.16b, v14.16b\n"
956 "smlal v20.4s, v12.4h, v21.4h\n"
957 "smlal v24.4s, v6.4h, v21.4h\n"
958 "sqrdmulh v20.4s, v20.4s, v8.4s\n"
959 "smlal v23.4s, v3.4h, v21.4h\n"
960 "smlal2 v5.4s, v12.8h, v21.8h\n"
961 "sqrdmulh v24.4s, v24.4s, v8.4s\n"
962 "smlal2 v22.4s, v6.8h, v21.8h\n"
963 "smlal2 v19.4s, v3.8h, v21.8h\n"
964 "sqrdmulh v23.4s, v23.4s, v8.4s\n"
965 "sshr v30.4s, v30.4s, #0x1f\n"
966 "and v3.16b, v20.16b, v28.16b\n"
967 "sqrdmulh v5.4s, v5.4s, v27.4s\n"
968 "and v25.16b, v24.16b, v28.16b\n"
969 "sqrdmulh v22.4s, v22.4s, v27.4s\n"
970 "and v16.16b, v23.16b, v28.16b\n"
971 "sqrdmulh v19.4s, v19.4s, v27.4s\n"
972 "sqadd v15.4s, v15.4s, v30.4s\n"
973 "sshr v3.4s, v3.4s, #0x1f\n"
974 "and v4.16b, v5.16b, v14.16b\n"
975 "sshr v25.4s, v25.4s, #0x1f\n"
976 "and v10.16b, v22.16b, v14.16b\n"
977 "sshr v16.4s, v16.4s, #0x1f\n"
978 "and v12.16b, v19.16b, v14.16b\n"
979 "sqadd v20.4s, v20.4s, v3.4s\n"
980 "sshr v4.4s, v4.4s, #0x1f\n"
981 "sqadd v24.4s, v24.4s, v25.4s\n"
982 "sshr v10.4s, v10.4s, #0x1f\n"
983 "sqadd v23.4s, v23.4s, v16.4s\n"
984 "sshr v12.4s, v12.4s, #0x1f\n"
985 "srshl v7.4s, v7.4s, v28.4s\n"
986 "srshl v20.4s, v20.4s, v28.4s\n"
987 "sqadd v5.4s, v5.4s, v4.4s\n"
988 "srshl v24.4s, v24.4s, v28.4s\n"
989 "sqadd v22.4s, v22.4s, v10.4s\n"
990 "srshl v23.4s, v23.4s, v28.4s\n"
991 "sqadd v19.4s, v19.4s, v12.4s\n"
992 "srshl v15.4s, v15.4s, v14.4s\n"
993 "sqxtn v7.4h, v7.4s\n"
994 "srshl v5.4s, v5.4s, v14.4s\n"
995 "sqxtn v20.4h, v20.4s\n"
996 "srshl v22.4s, v22.4s, v14.4s\n"
997 "sqxtn v24.4h, v24.4s\n"
998 "srshl v19.4s, v19.4s, v14.4s\n"
999 "sqxtn v23.4h, v23.4s\n"
1000 "sqxtn2 v7.8h, v15.4s\n"
1001 "sqxtn2 v20.8h, v5.4s\n"
1002 "sqxtn2 v24.8h, v22.4s\n"
1003 "sqxtn2 v23.8h, v19.4s\n"
1004 "sqadd v7.8h, v7.8h, v26.8h\n"
1005 "sqadd v20.8h, v20.8h, v26.8h\n"
1006 "sqadd v24.8h, v24.8h, v26.8h\n"
1007 "sqadd v23.8h, v23.8h, v26.8h\n"
1008 "smax v7.8h, v7.8h, v11.8h\n"
1009 "smax v20.8h, v20.8h, v11.8h\n"
1010 "smax v24.8h, v24.8h, v11.8h\n"
1011 "smax v23.8h, v23.8h, v11.8h\n"
1012 "smin v7.8h, v7.8h, v0.8h\n"
1013 "smin v20.8h, v20.8h, v0.8h\n"
1014 "smin v24.8h, v24.8h, v0.8h\n"
1015 "smin v23.8h, v23.8h, v0.8h\n"
1016 "uzp1 v7.16b, v7.16b, v7.16b\n"
1017 "str d7, [x17, x4]\n"
1018 "uzp1 v20.16b, v20.16b, v20.16b\n"
1019 "uzp1 v24.16b, v24.16b, v24.16b\n"
1020 "str d20, [x16, x4]\n"
1021 "uzp1 v23.16b, v23.16b, v23.16b\n"
1022 "str d24, [x15, x4]\n"
1023 "str d23, [x14, x4]\n"
1024 "add x4, x4, #0x8\n"
1026 "add x6, x6, #0xc8\n"
1028 "ldr x20, [%x[params], %[offsetof_Params_bias]]\n"
1030 "ld1 { v7.4s }, [x20], #0x10\n"
1032 "ld1 { v15.d }[0], [x20], #0x8\n"
1034 "ld1 { v15.s }[2], [x20]\n"
1038 "ld1 { v15.s }[0], [x20]\n"
1042 "ld1 { v7.d }[0], [x20], #0x8\n"
1044 "ld1 { v7.s }[2], [x20]\n"
1048 "ld1 { v7.s }[0], [x20]\n"
1050 "ldr d6, [x6, #0x0]\n"
1051 "ldr d14, [x6, #0x8]\n"
1052 "mov v20.16b, v7.16b\n"
1053 "mov v5.16b, v15.16b\n"
1054 "ldr d10, [x6, #0x10]\n"
1055 "ldr d21, [x6, #0x18]\n"
1056 "mov v24.16b, v7.16b\n"
1057 "mov v22.16b, v15.16b\n"
1058 "ldr d12, [x6, #0x20]\n"
1059 "ldp x9, x28, [x5, #0x0]\n"
1060 "mov v23.16b, v7.16b\n"
1061 "mov v19.16b, v15.16b\n"
1062 "ldp x27, x26, [x5, #0x10]\n"
1063 "ldp x25, x24, [x5, #0x20]\n"
1064 "ssubl v6.8h, v6.8b, v13.8b\n"
1065 "ssubl v14.8h, v14.8b, v13.8b\n"
1066 "ldp x23, x22, [x5, #0x30]\n"
1067 "ldp x21, x20, [x5, #0x40]\n"
1068 "ssubl v10.8h, v10.8b, v13.8b\n"
1069 "ssubl v21.8h, v21.8b, v13.8b\n"
1070 "ssubl v12.8h, v12.8b, v13.8b\n"
1072 "add x28, x28, x3\n"
1073 "add x27, x27, x3\n"
1074 "add x26, x26, x3\n"
1075 "add x25, x25, x3\n"
1076 "add x24, x24, x3\n"
1077 "add x23, x23, x3\n"
1078 "add x22, x22, x3\n"
1079 "add x21, x21, x3\n"
1080 "add x20, x20, x3\n"
1082 "ld1 { v31.s }[0], [x9], #0x4\n"
1083 "ld1 { v17.s }[0], [x28], #0x4\n"
1084 "ld1 { v30.s }[0], [x27], #0x4\n"
1085 "ld1 { v16.s }[0], [x26], #0x4\n"
1086 "ld1 { v3.s }[0], [x25], #0x4\n"
1087 "ld1 { v4.s }[0], [x24], #0x4\n"
1088 "ld1 { v25.s }[0], [x23], #0x4\n"
1089 "ld1 { v9.s }[0], [x22], #0x4\n"
1090 "ld1 { v29.s }[0], [x21], #0x4\n"
1091 "ld1 { v28.s }[0], [x20], #0x4\n"
1093 "ld1 { v31.h }[2], [x9], #0x2\n"
1094 "ld1 { v17.h }[2], [x28], #0x2\n"
1095 "ld1 { v30.h }[2], [x27], #0x2\n"
1096 "ld1 { v16.h }[2], [x26], #0x2\n"
1097 "ld1 { v3.h }[2], [x25], #0x2\n"
1098 "ld1 { v4.h }[2], [x24], #0x2\n"
1099 "ld1 { v25.h }[2], [x23], #0x2\n"
1100 "ld1 { v9.h }[2], [x22], #0x2\n"
1101 "ld1 { v29.h }[2], [x21], #0x2\n"
1102 "ld1 { v28.h }[2], [x20], #0x2\n"
1104 "ld1 { v31.b }[6], [x9]\n"
1105 "ld1 { v17.b }[6], [x28]\n"
1106 "ld1 { v30.b }[6], [x27]\n"
1107 "ld1 { v16.b }[6], [x26]\n"
1108 "ld1 { v3.b }[6], [x25]\n"
1109 "ld1 { v4.b }[6], [x24]\n"
1110 "ld1 { v25.b }[6], [x23]\n"
1111 "ld1 { v9.b }[6], [x22]\n"
1112 "ld1 { v29.b }[6], [x21]\n"
1113 "ld1 { v28.b }[6], [x20]\n"
1117 "ld1 { v31.b }[4], [x9]\n"
1118 "ld1 { v17.b }[4], [x28]\n"
1119 "ld1 { v30.b }[4], [x27]\n"
1120 "ld1 { v16.b }[4], [x26]\n"
1121 "ld1 { v3.b }[4], [x25]\n"
1122 "ld1 { v4.b }[4], [x24]\n"
1123 "ld1 { v25.b }[4], [x23]\n"
1124 "ld1 { v9.b }[4], [x22]\n"
1125 "ld1 { v29.b }[4], [x21]\n"
1126 "ld1 { v28.b }[4], [x20]\n"
1130 "ld1 { v31.h }[0], [x9], #0x2\n"
1131 "ld1 { v17.h }[0], [x28], #0x2\n"
1132 "ld1 { v30.h }[0], [x27], #0x2\n"
1133 "ld1 { v16.h }[0], [x26], #0x2\n"
1134 "ld1 { v3.h }[0], [x25], #0x2\n"
1135 "ld1 { v4.h }[0], [x24], #0x2\n"
1136 "ld1 { v25.h }[0], [x23], #0x2\n"
1137 "ld1 { v9.h }[0], [x22], #0x2\n"
1138 "ld1 { v29.h }[0], [x21], #0x2\n"
1139 "ld1 { v28.h }[0], [x20], #0x2\n"
1141 "ld1 { v31.b }[2], [x9]\n"
1142 "ld1 { v17.b }[2], [x28]\n"
1143 "ld1 { v30.b }[2], [x27]\n"
1144 "ld1 { v16.b }[2], [x26]\n"
1145 "ld1 { v3.b }[2], [x25]\n"
1146 "ld1 { v4.b }[2], [x24]\n"
1147 "ld1 { v25.b }[2], [x23]\n"
1148 "ld1 { v9.b }[2], [x22]\n"
1149 "ld1 { v29.b }[2], [x21]\n"
1150 "ld1 { v28.b }[2], [x20]\n"
1154 "ld1 { v31.b }[0], [x9]\n"
1155 "ld1 { v17.b }[0], [x28]\n"
1156 "ld1 { v30.b }[0], [x27]\n"
1157 "ld1 { v16.b }[0], [x26]\n"
1158 "ld1 { v3.b }[0], [x25]\n"
1159 "ld1 { v4.b }[0], [x24]\n"
1160 "ld1 { v25.b }[0], [x23]\n"
1161 "ld1 { v9.b }[0], [x22]\n"
1162 "ld1 { v29.b }[0], [x21]\n"
1163 "ld1 { v28.b }[0], [x20]\n"
1165 "usubl v31.8h, v31.8b, v18.8b\n"
1166 "usubl v17.8h, v17.8b, v18.8b\n"
1167 "smlal v7.4s, v31.4h, v6.4h\n"
1168 "ldr x20, [x5, #0x50]\n"
1169 "usubl v30.8h, v30.8b, v18.8b\n"
1170 "smlal2 v15.4s, v31.8h, v6.8h\n"
1171 "smlal v20.4s, v17.4h, v6.4h\n"
1172 "smlal2 v5.4s, v17.8h, v6.8h\n"
1173 "smlal v24.4s, v30.4h, v6.4h\n"
1174 "usubl v16.8h, v16.8b, v18.8b\n"
1175 "add x20, x20, x3\n"
1176 "smlal2 v22.4s, v30.8h, v6.8h\n"
1177 "usubl v3.8h, v3.8b, v18.8b\n"
1178 "smlal v23.4s, v16.4h, v6.4h\n"
1179 "smlal2 v19.4s, v16.8h, v6.8h\n"
1180 "smlal v7.4s, v17.4h, v14.4h\n"
1181 "usubl v4.8h, v4.8b, v18.8b\n"
1182 "smlal2 v15.4s, v17.8h, v14.8h\n"
1183 "smlal v20.4s, v3.4h, v14.4h\n"
1184 "usubl v25.8h, v25.8b, v18.8b\n"
1185 "smlal2 v5.4s, v3.8h, v14.8h\n"
1186 "smlal v24.4s, v16.4h, v14.4h\n"
1187 "usubl v9.8h, v9.8b, v18.8b\n"
1188 "smlal2 v22.4s, v16.8h, v14.8h\n"
1189 "usubl v29.8h, v29.8b, v18.8b\n"
1190 "smlal v23.4s, v4.4h, v14.4h\n"
1191 "usubl v28.8h, v28.8b, v18.8b\n"
1192 "smlal2 v19.4s, v4.8h, v14.8h\n"
1193 "smlal v7.4s, v3.4h, v10.4h\n"
1194 "smlal2 v15.4s, v3.8h, v10.8h\n"
1195 "smlal v20.4s, v25.4h, v10.4h\n"
1196 "smlal2 v5.4s, v25.8h, v10.8h\n"
1197 "smlal v24.4s, v4.4h, v10.4h\n"
1198 "smlal2 v22.4s, v4.8h, v10.8h\n"
1200 "ld1 { v27.s }[0], [x20], #0x4\n"
1202 "ld1 { v27.h }[2], [x20], #0x2\n"
1204 "ld1 { v27.b }[6], [x20]\n"
1208 "ld1 { v27.b }[4], [x20]\n"
1212 "ld1 { v27.h }[0], [x20], #0x2\n"
1214 "ld1 { v27.b }[2], [x20]\n"
1218 "ld1 { v27.b }[0], [x20]\n"
1220 "usubl v27.8h, v27.8b, v18.8b\n"
1221 "ldr x20, [x5, #0x58]\n"
1222 "smlal v23.4s, v27.4h, v10.4h\n"
1223 "smlal2 v19.4s, v27.8h, v10.8h\n"
1224 "smlal v7.4s, v25.4h, v21.4h\n"
1225 "smlal2 v15.4s, v25.8h, v21.8h\n"
1226 "add x20, x20, x3\n"
1227 "smlal v20.4s, v9.4h, v21.4h\n"
1228 "smlal2 v5.4s, v9.8h, v21.8h\n"
1229 "smlal v24.4s, v27.4h, v21.4h\n"
1230 "smlal2 v22.4s, v27.8h, v21.8h\n"
1232 "ld1 { v6.s }[0], [x20], #0x4\n"
1234 "ld1 { v6.h }[2], [x20], #0x2\n"
1236 "ld1 { v6.b }[6], [x20]\n"
1240 "ld1 { v6.b }[4], [x20]\n"
1244 "ld1 { v6.h }[0], [x20], #0x2\n"
1246 "ld1 { v6.b }[2], [x20]\n"
1250 "ld1 { v6.b }[0], [x20]\n"
1252 "usubl v6.8h, v6.8b, v18.8b\n"
1253 "ldr x20, [x5, #0x60]\n"
1254 "smlal v23.4s, v6.4h, v21.4h\n"
1255 "smlal2 v19.4s, v6.8h, v21.8h\n"
1256 "smlal v7.4s, v9.4h, v12.4h\n"
1257 "smlal2 v15.4s, v9.8h, v12.8h\n"
1258 "add x20, x20, x3\n"
1260 "ld1 { v9.s }[0], [x20], #0x4\n"
1262 "ld1 { v9.h }[2], [x20], #0x2\n"
1264 "ld1 { v9.b }[6], [x20]\n"
1268 "ld1 { v9.b }[4], [x20]\n"
1272 "ld1 { v9.h }[0], [x20], #0x2\n"
1274 "ld1 { v9.b }[2], [x20]\n"
1278 "ld1 { v9.b }[0], [x20]\n"
1280 "ldr d14, [x6, #0x28]\n"
1281 "usubl v9.8h, v9.8b, v18.8b\n"
1282 "smlal v20.4s, v9.4h, v12.4h\n"
1283 "smlal2 v5.4s, v9.8h, v12.8h\n"
1284 "smlal v24.4s, v6.4h, v12.4h\n"
1285 "smlal2 v22.4s, v6.8h, v12.8h\n"
1286 "ssubl v14.8h, v14.8b, v13.8b\n"
1287 "ldr x20, [x5, #0x68]\n"
1288 "smlal v23.4s, v29.4h, v12.4h\n"
1289 "smlal2 v19.4s, v29.8h, v12.8h\n"
1290 "add x20, x20, x3\n"
1291 "smlal v7.4s, v30.4h, v14.4h\n"
1292 "smlal2 v15.4s, v30.8h, v14.8h\n"
1293 "smlal v20.4s, v16.4h, v14.4h\n"
1294 "smlal2 v5.4s, v16.8h, v14.8h\n"
1295 "smlal v24.4s, v28.4h, v14.4h\n"
1296 "smlal2 v22.4s, v28.8h, v14.8h\n"
1298 "ld1 { v25.s }[0], [x20], #0x4\n"
1300 "ld1 { v25.h }[2], [x20], #0x2\n"
1302 "ld1 { v25.b }[6], [x20]\n"
1306 "ld1 { v25.b }[4], [x20]\n"
1310 "ld1 { v25.h }[0], [x20], #0x2\n"
1312 "ld1 { v25.b }[2], [x20]\n"
1316 "ld1 { v25.b }[0], [x20]\n"
1318 "ldr d21, [x6, #0x30]\n"
1319 "usubl v25.8h, v25.8b, v18.8b\n"
1320 "ssubl v21.8h, v21.8b, v13.8b\n"
1321 "ldr x20, [x5, #0x70]\n"
1322 "smlal v23.4s, v25.4h, v14.4h\n"
1323 "smlal2 v19.4s, v25.8h, v14.8h\n"
1324 "add x20, x20, x3\n"
1325 "smlal v7.4s, v16.4h, v21.4h\n"
1326 "smlal2 v15.4s, v16.8h, v21.8h\n"
1327 "smlal v20.4s, v4.4h, v21.4h\n"
1328 "smlal2 v5.4s, v4.8h, v21.8h\n"
1329 "smlal v24.4s, v25.4h, v21.4h\n"
1330 "smlal2 v22.4s, v25.8h, v21.8h\n"
1332 "ld1 { v10.s }[0], [x20], #0x4\n"
1334 "ld1 { v10.h }[2], [x20], #0x2\n"
1336 "ld1 { v10.b }[6], [x20]\n"
1340 "ld1 { v10.b }[4], [x20]\n"
1344 "ld1 { v10.h }[0], [x20], #0x2\n"
1346 "ld1 { v10.b }[2], [x20]\n"
1350 "ld1 { v10.b }[0], [x20]\n"
1352 "ldr d9, [x6, #0x38]\n"
1353 "usubl v10.8h, v10.8b, v18.8b\n"
1354 "ssubl v9.8h, v9.8b, v13.8b\n"
1355 "ldr x20, [x5, #0x78]\n"
1356 "smlal v23.4s, v10.4h, v21.4h\n"
1357 "smlal2 v19.4s, v10.8h, v21.8h\n"
1358 "add x20, x20, x3\n"
1359 "smlal v7.4s, v4.4h, v9.4h\n"
1360 "smlal2 v15.4s, v4.8h, v9.8h\n"
1361 "smlal v20.4s, v27.4h, v9.4h\n"
1362 "smlal2 v5.4s, v27.8h, v9.8h\n"
1363 "smlal v24.4s, v10.4h, v9.4h\n"
1364 "smlal2 v22.4s, v10.8h, v9.8h\n"
1366 "ld1 { v12.s }[0], [x20], #0x4\n"
1368 "ld1 { v12.h }[2], [x20], #0x2\n"
1370 "ld1 { v12.b }[6], [x20]\n"
1374 "ld1 { v12.b }[4], [x20]\n"
1378 "ld1 { v12.h }[0], [x20], #0x2\n"
1380 "ld1 { v12.b }[2], [x20]\n"
1384 "ld1 { v12.b }[0], [x20]\n"
1386 "ldr d31, [x6, #0x40]\n"
1387 "usubl v12.8h, v12.8b, v18.8b\n"
1388 "ssubl v31.8h, v31.8b, v13.8b\n"
1389 "ldr x20, [x5, #0x80]\n"
1390 "smlal v23.4s, v12.4h, v9.4h\n"
1391 "smlal2 v19.4s, v12.8h, v9.8h\n"
1392 "add x20, x20, x3\n"
1393 "smlal v7.4s, v27.4h, v31.4h\n"
1394 "smlal2 v15.4s, v27.8h, v31.8h\n"
1395 "smlal v20.4s, v6.4h, v31.4h\n"
1396 "smlal2 v5.4s, v6.8h, v31.8h\n"
1397 "smlal v24.4s, v12.4h, v31.4h\n"
1398 "smlal2 v22.4s, v12.8h, v31.8h\n"
1400 "ld1 { v8.s }[0], [x20], #0x4\n"
1402 "ld1 { v8.h }[2], [x20], #0x2\n"
1404 "ld1 { v8.b }[6], [x20]\n"
1408 "ld1 { v8.b }[4], [x20]\n"
1412 "ld1 { v8.h }[0], [x20], #0x2\n"
1414 "ld1 { v8.b }[2], [x20]\n"
1418 "ld1 { v8.b }[0], [x20]\n"
1420 "ldr d16, [x6, #0x48]\n"
1421 "usubl v8.8h, v8.8b, v18.8b\n"
1422 "ssubl v16.8h, v16.8b, v13.8b\n"
1423 "ldr x20, [x5, #0x88]\n"
1424 "smlal v23.4s, v8.4h, v31.4h\n"
1425 "smlal2 v19.4s, v8.8h, v31.8h\n"
1426 "add x20, x20, x3\n"
1427 "smlal v7.4s, v6.4h, v16.4h\n"
1428 "smlal2 v15.4s, v6.8h, v16.8h\n"
1429 "smlal v20.4s, v29.4h, v16.4h\n"
1430 "smlal2 v5.4s, v29.8h, v16.8h\n"
1431 "smlal v24.4s, v8.4h, v16.4h\n"
1432 "smlal2 v22.4s, v8.8h, v16.8h\n"
1434 "ld1 { v27.s }[0], [x20], #0x4\n"
1436 "ld1 { v27.h }[2], [x20], #0x2\n"
1438 "ld1 { v27.b }[6], [x20]\n"
1442 "ld1 { v27.b }[4], [x20]\n"
1446 "ld1 { v27.h }[0], [x20], #0x2\n"
1448 "ld1 { v27.b }[2], [x20]\n"
1452 "ld1 { v27.b }[0], [x20]\n"
1454 "ldr d21, [x6, #0x50]\n"
1455 "usubl v27.8h, v27.8b, v18.8b\n"
1456 "ssubl v21.8h, v21.8b, v13.8b\n"
1457 "ldr x20, [x5, #0x90]\n"
1458 "smlal v23.4s, v27.4h, v16.4h\n"
1459 "smlal2 v19.4s, v27.8h, v16.8h\n"
1460 "add x20, x20, x3\n"
1461 "smlal v7.4s, v28.4h, v21.4h\n"
1462 "smlal2 v15.4s, v28.8h, v21.8h\n"
1463 "smlal v20.4s, v25.4h, v21.4h\n"
1464 "smlal2 v5.4s, v25.8h, v21.8h\n"
1466 "ld1 { v31.s }[0], [x20], #0x4\n"
1468 "ld1 { v31.h }[2], [x20], #0x2\n"
1470 "ld1 { v31.b }[6], [x20]\n"
1474 "ld1 { v31.b }[4], [x20]\n"
1478 "ld1 { v31.h }[0], [x20], #0x2\n"
1480 "ld1 { v31.b }[2], [x20]\n"
1484 "ld1 { v31.b }[0], [x20]\n"
1486 "usubl v31.8h, v31.8b, v18.8b\n"
1487 "ldr x20, [x5, #0x98]\n"
1488 "smlal v24.4s, v31.4h, v21.4h\n"
1489 "smlal2 v22.4s, v31.8h, v21.8h\n"
1490 "add x20, x20, x3\n"
1492 "ld1 { v28.s }[0], [x20], #0x4\n"
1494 "ld1 { v28.h }[2], [x20], #0x2\n"
1496 "ld1 { v28.b }[6], [x20]\n"
1500 "ld1 { v28.b }[4], [x20]\n"
1504 "ld1 { v28.h }[0], [x20], #0x2\n"
1506 "ld1 { v28.b }[2], [x20]\n"
1510 "ld1 { v28.b }[0], [x20]\n"
1512 "ldr d2, [x6, #0x58]\n"
1513 "usubl v28.8h, v28.8b, v18.8b\n"
1514 "ssubl v2.8h, v2.8b, v13.8b\n"
1515 "ldr x20, [x5, #0xa0]\n"
1516 "smlal v23.4s, v28.4h, v21.4h\n"
1517 "smlal2 v19.4s, v28.8h, v21.8h\n"
1518 "add x20, x20, x3\n"
1519 "smlal v7.4s, v25.4h, v2.4h\n"
1520 "smlal2 v15.4s, v25.8h, v2.8h\n"
1521 "smlal v20.4s, v10.4h, v2.4h\n"
1522 "smlal2 v5.4s, v10.8h, v2.8h\n"
1523 "smlal v24.4s, v28.4h, v2.4h\n"
1524 "smlal2 v22.4s, v28.8h, v2.8h\n"
1526 "ld1 { v21.s }[0], [x20], #0x4\n"
1528 "ld1 { v21.h }[2], [x20], #0x2\n"
1530 "ld1 { v21.b }[6], [x20]\n"
1534 "ld1 { v21.b }[4], [x20]\n"
1538 "ld1 { v21.h }[0], [x20], #0x2\n"
1540 "ld1 { v21.b }[2], [x20]\n"
1544 "ld1 { v21.b }[0], [x20]\n"
1546 "ldr d25, [x6, #0x60]\n"
1547 "usubl v21.8h, v21.8b, v18.8b\n"
1548 "ssubl v25.8h, v25.8b, v13.8b\n"
1549 "ldr x20, [x5, #0xa8]\n"
1550 "smlal v23.4s, v21.4h, v2.4h\n"
1551 "smlal2 v19.4s, v21.8h, v2.8h\n"
1552 "add x20, x20, x3\n"
1553 "smlal v7.4s, v10.4h, v25.4h\n"
1554 "smlal2 v15.4s, v10.8h, v25.8h\n"
1555 "smlal v20.4s, v12.4h, v25.4h\n"
1556 "smlal2 v5.4s, v12.8h, v25.8h\n"
1557 "smlal v24.4s, v21.4h, v25.4h\n"
1558 "smlal2 v22.4s, v21.8h, v25.8h\n"
1560 "ld1 { v9.s }[0], [x20], #0x4\n"
1562 "ld1 { v9.h }[2], [x20], #0x2\n"
1564 "ld1 { v9.b }[6], [x20]\n"
1568 "ld1 { v9.b }[4], [x20]\n"
1572 "ld1 { v9.h }[0], [x20], #0x2\n"
1574 "ld1 { v9.b }[2], [x20]\n"
1578 "ld1 { v9.b }[0], [x20]\n"
1580 "ldr d1, [x6, #0x68]\n"
1581 "usubl v9.8h, v9.8b, v18.8b\n"
1582 "ssubl v1.8h, v1.8b, v13.8b\n"
1583 "ldr x20, [x5, #0xb0]\n"
1584 "smlal v23.4s, v9.4h, v25.4h\n"
1585 "smlal2 v19.4s, v9.8h, v25.8h\n"
1586 "add x20, x20, x3\n"
1587 "smlal v7.4s, v12.4h, v1.4h\n"
1588 "smlal2 v15.4s, v12.8h, v1.8h\n"
1589 "smlal v20.4s, v8.4h, v1.4h\n"
1590 "smlal2 v5.4s, v8.8h, v1.8h\n"
1591 "smlal v24.4s, v9.4h, v1.4h\n"
1592 "smlal2 v22.4s, v9.8h, v1.8h\n"
1594 "ld1 { v3.s }[0], [x20], #0x4\n"
1596 "ld1 { v3.h }[2], [x20], #0x2\n"
1598 "ld1 { v3.b }[6], [x20]\n"
1602 "ld1 { v3.b }[4], [x20]\n"
1606 "ld1 { v3.h }[0], [x20], #0x2\n"
1608 "ld1 { v3.b }[2], [x20]\n"
1612 "ld1 { v3.b }[0], [x20]\n"
1614 "ldr d16, [x6, #0x70]\n"
1615 "usubl v3.8h, v3.8b, v18.8b\n"
1616 "ssubl v16.8h, v16.8b, v13.8b\n"
1617 "ldr x20, [x5, #0xb8]\n"
1618 "smlal v23.4s, v3.4h, v1.4h\n"
1619 "smlal2 v19.4s, v3.8h, v1.8h\n"
1620 "add x20, x20, x3\n"
1621 "smlal v7.4s, v8.4h, v16.4h\n"
1622 "smlal2 v15.4s, v8.8h, v16.8h\n"
1623 "smlal v20.4s, v27.4h, v16.4h\n"
1624 "smlal2 v5.4s, v27.8h, v16.8h\n"
1625 "smlal v24.4s, v3.4h, v16.4h\n"
1626 "smlal2 v22.4s, v3.8h, v16.8h\n"
1628 "ld1 { v14.s }[0], [x20], #0x4\n"
1630 "ld1 { v14.h }[2], [x20], #0x2\n"
1632 "ld1 { v14.b }[6], [x20]\n"
1636 "ld1 { v14.b }[4], [x20]\n"
1640 "ld1 { v14.h }[0], [x20], #0x2\n"
1642 "ld1 { v14.b }[2], [x20]\n"
1646 "ld1 { v14.b }[0], [x20]\n"
1648 "ldr d17, [x6, #0x78]\n"
1649 "usubl v14.8h, v14.8b, v18.8b\n"
1650 "ssubl v17.8h, v17.8b, v13.8b\n"
1651 "ldr x20, [x5, #0xc0]\n"
1652 "smlal v23.4s, v14.4h, v16.4h\n"
1653 "smlal2 v19.4s, v14.8h, v16.8h\n"
1654 "add x20, x20, x3\n"
1655 "smlal v7.4s, v31.4h, v17.4h\n"
1656 "smlal2 v15.4s, v31.8h, v17.8h\n"
1657 "smlal v20.4s, v28.4h, v17.4h\n"
1658 "smlal2 v5.4s, v28.8h, v17.8h\n"
1660 "ld1 { v1.s }[0], [x20], #0x4\n"
1662 "ld1 { v1.h }[2], [x20], #0x2\n"
1664 "ld1 { v1.b }[6], [x20]\n"
1668 "ld1 { v1.b }[4], [x20]\n"
1672 "ld1 { v1.h }[0], [x20], #0x2\n"
1674 "ld1 { v1.b }[2], [x20]\n"
1678 "ld1 { v1.b }[0], [x20]\n"
1680 "usubl v1.8h, v1.8b, v18.8b\n"
1681 "ldr x20, [x5, #0xc8]\n"
1682 "smlal v24.4s, v1.4h, v17.4h\n"
1683 "smlal2 v22.4s, v1.8h, v17.8h\n"
1684 "add x20, x20, x3\n"
1686 "ld1 { v16.s }[0], [x20], #0x4\n"
1688 "ld1 { v16.h }[2], [x20], #0x2\n"
1690 "ld1 { v16.b }[6], [x20]\n"
1694 "ld1 { v16.b }[4], [x20]\n"
1698 "ld1 { v16.h }[0], [x20], #0x2\n"
1700 "ld1 { v16.b }[2], [x20]\n"
1704 "ld1 { v16.b }[0], [x20]\n"
1706 "ldr d29, [x6, #0x80]\n"
1707 "usubl v16.8h, v16.8b, v18.8b\n"
1708 "ssubl v29.8h, v29.8b, v13.8b\n"
1709 "ldr x20, [x5, #0xd0]\n"
1710 "smlal v23.4s, v16.4h, v17.4h\n"
1711 "smlal2 v19.4s, v16.8h, v17.8h\n"
1712 "add x20, x20, x3\n"
1713 "smlal v7.4s, v28.4h, v29.4h\n"
1714 "smlal2 v15.4s, v28.8h, v29.8h\n"
1715 "smlal v20.4s, v21.4h, v29.4h\n"
1716 "smlal2 v5.4s, v21.8h, v29.8h\n"
1717 "smlal v24.4s, v16.4h, v29.4h\n"
1718 "smlal2 v22.4s, v16.8h, v29.8h\n"
1720 "ld1 { v30.s }[0], [x20], #0x4\n"
1722 "ld1 { v30.h }[2], [x20], #0x2\n"
1724 "ld1 { v30.b }[6], [x20]\n"
1728 "ld1 { v30.b }[4], [x20]\n"
1732 "ld1 { v30.h }[0], [x20], #0x2\n"
1734 "ld1 { v30.b }[2], [x20]\n"
1738 "ld1 { v30.b }[0], [x20]\n"
1740 "ldr d12, [x6, #0x88]\n"
1741 "usubl v30.8h, v30.8b, v18.8b\n"
1742 "ssubl v12.8h, v12.8b, v13.8b\n"
1743 "ldr x20, [x5, #0xd8]\n"
1744 "smlal v23.4s, v30.4h, v29.4h\n"
1745 "smlal2 v19.4s, v30.8h, v29.8h\n"
1746 "add x20, x20, x3\n"
1747 "smlal v7.4s, v21.4h, v12.4h\n"
1748 "smlal2 v15.4s, v21.8h, v12.8h\n"
1749 "smlal v20.4s, v9.4h, v12.4h\n"
1750 "smlal2 v5.4s, v9.8h, v12.8h\n"
1751 "smlal v24.4s, v30.4h, v12.4h\n"
1752 "smlal2 v22.4s, v30.8h, v12.8h\n"
1754 "ld1 { v29.s }[0], [x20], #0x4\n"
1756 "ld1 { v29.h }[2], [x20], #0x2\n"
1758 "ld1 { v29.b }[6], [x20]\n"
1762 "ld1 { v29.b }[4], [x20]\n"
1766 "ld1 { v29.h }[0], [x20], #0x2\n"
1768 "ld1 { v29.b }[2], [x20]\n"
1772 "ld1 { v29.b }[0], [x20]\n"
1774 "ldr d21, [x6, #0x90]\n"
1775 "usubl v29.8h, v29.8b, v18.8b\n"
1776 "ssubl v21.8h, v21.8b, v13.8b\n"
1777 "ldr x20, [x5, #0xe0]\n"
1778 "smlal v23.4s, v29.4h, v12.4h\n"
1779 "smlal2 v19.4s, v29.8h, v12.8h\n"
1780 "add x20, x20, x3\n"
1781 "smlal v7.4s, v9.4h, v21.4h\n"
1782 "smlal2 v15.4s, v9.8h, v21.8h\n"
1783 "smlal v20.4s, v3.4h, v21.4h\n"
1784 "smlal2 v5.4s, v3.8h, v21.8h\n"
1785 "smlal v24.4s, v29.4h, v21.4h\n"
1786 "smlal2 v22.4s, v29.8h, v21.8h\n"
1788 "ld1 { v25.s }[0], [x20], #0x4\n"
1790 "ld1 { v25.h }[2], [x20], #0x2\n"
1792 "ld1 { v25.b }[6], [x20]\n"
1796 "ld1 { v25.b }[4], [x20]\n"
1800 "ld1 { v25.h }[0], [x20], #0x2\n"
1802 "ld1 { v25.b }[2], [x20]\n"
1806 "ld1 { v25.b }[0], [x20]\n"
1808 "ldr d8, [x6, #0x98]\n"
1809 "usubl v25.8h, v25.8b, v18.8b\n"
1810 "ssubl v8.8h, v8.8b, v13.8b\n"
1811 "ldr x20, [x5, #0xe8]\n"
1812 "smlal v23.4s, v25.4h, v21.4h\n"
1813 "smlal2 v19.4s, v25.8h, v21.8h\n"
1814 "add x20, x20, x3\n"
1815 "smlal v7.4s, v3.4h, v8.4h\n"
1816 "smlal2 v15.4s, v3.8h, v8.8h\n"
1817 "smlal v20.4s, v14.4h, v8.4h\n"
1818 "smlal2 v5.4s, v14.8h, v8.8h\n"
1819 "smlal v24.4s, v25.4h, v8.4h\n"
1820 "smlal2 v22.4s, v25.8h, v8.8h\n"
1822 "ld1 { v21.s }[0], [x20], #0x4\n"
1824 "ld1 { v21.h }[2], [x20], #0x2\n"
1826 "ld1 { v21.b }[6], [x20]\n"
1830 "ld1 { v21.b }[4], [x20]\n"
1834 "ld1 { v21.h }[0], [x20], #0x2\n"
1836 "ld1 { v21.b }[2], [x20]\n"
1840 "ld1 { v21.b }[0], [x20]\n"
1842 "ldr d9, [x6, #0xa0]\n"
1843 "usubl v21.8h, v21.8b, v18.8b\n"
1844 "ssubl v9.8h, v9.8b, v13.8b\n"
1845 "ldr x20, [x5, #0xf0]\n"
1846 "smlal v23.4s, v21.4h, v8.4h\n"
1847 "smlal2 v19.4s, v21.8h, v8.8h\n"
1848 "add x20, x20, x3\n"
1849 "smlal v7.4s, v1.4h, v9.4h\n"
1850 "smlal2 v15.4s, v1.8h, v9.8h\n"
1851 "smlal v20.4s, v16.4h, v9.4h\n"
1852 "smlal2 v5.4s, v16.8h, v9.8h\n"
1854 "ld1 { v12.s }[0], [x20], #0x4\n"
1856 "ld1 { v12.h }[2], [x20], #0x2\n"
1858 "ld1 { v12.b }[6], [x20]\n"
1862 "ld1 { v12.b }[4], [x20]\n"
1866 "ld1 { v12.h }[0], [x20], #0x2\n"
1868 "ld1 { v12.b }[2], [x20]\n"
1872 "ld1 { v12.b }[0], [x20]\n"
1874 "usubl v12.8h, v12.8b, v18.8b\n"
1875 "ldr x20, [x5, #0xf8]\n"
1876 "smlal v24.4s, v12.4h, v9.4h\n"
1877 "smlal2 v22.4s, v12.8h, v9.8h\n"
1878 "add x20, x20, x3\n"
1880 "ld1 { v10.s }[0], [x20], #0x4\n"
1882 "ld1 { v10.h }[2], [x20], #0x2\n"
1884 "ld1 { v10.b }[6], [x20]\n"
1888 "ld1 { v10.b }[4], [x20]\n"
1892 "ld1 { v10.h }[0], [x20], #0x2\n"
1894 "ld1 { v10.b }[2], [x20]\n"
1898 "ld1 { v10.b }[0], [x20]\n"
1900 "ldr d12, [x6, #0xa8]\n"
1901 "usubl v10.8h, v10.8b, v18.8b\n"
1902 "ssubl v12.8h, v12.8b, v13.8b\n"
1903 "ldr x20, [x5, #0x100]\n"
1904 "smlal v23.4s, v10.4h, v9.4h\n"
1905 "smlal2 v19.4s, v10.8h, v9.8h\n"
1906 "add x20, x20, x3\n"
1907 "smlal v7.4s, v16.4h, v12.4h\n"
1908 "smlal2 v15.4s, v16.8h, v12.8h\n"
1909 "smlal v20.4s, v30.4h, v12.4h\n"
1910 "smlal2 v5.4s, v30.8h, v12.8h\n"
1911 "smlal v24.4s, v10.4h, v12.4h\n"
1912 "smlal2 v22.4s, v10.8h, v12.8h\n"
1913 "tbz x1, #2, 101f\n"
1914 "ld1 { v9.s }[0], [x20], #0x4\n"
1915 "tbz x1, #1, 100f\n"
1916 "ld1 { v9.h }[2], [x20], #0x2\n"
1917 "tbz x1, #0, 103f\n"
1918 "ld1 { v9.b }[6], [x20]\n"
1921 "tbz x1, #0, 103f\n"
1922 "ld1 { v9.b }[4], [x20]\n"
1925 "tbz x1, #1, 102f\n"
1926 "ld1 { v9.h }[0], [x20], #0x2\n"
1927 "tbz x1, #0, 103f\n"
1928 "ld1 { v9.b }[2], [x20]\n"
1931 "tbz x1, #0, 103f\n"
1932 "ld1 { v9.b }[0], [x20]\n"
1934 "ldr d28, [x6, #0xb0]\n"
1935 "usubl v9.8h, v9.8b, v18.8b\n"
1936 "ssubl v28.8h, v28.8b, v13.8b\n"
1937 "ldr x20, [x5, #0x108]\n"
1938 "smlal v23.4s, v9.4h, v12.4h\n"
1939 "smlal2 v19.4s, v9.8h, v12.8h\n"
1940 "add x20, x20, x3\n"
1941 "smlal v7.4s, v30.4h, v28.4h\n"
1942 "smlal2 v15.4s, v30.8h, v28.8h\n"
1943 "smlal v20.4s, v29.4h, v28.4h\n"
1944 "smlal2 v5.4s, v29.8h, v28.8h\n"
1945 "smlal v24.4s, v9.4h, v28.4h\n"
1946 "smlal2 v22.4s, v9.8h, v28.8h\n"
1947 "tbz x1, #2, 105f\n"
1948 "ld1 { v2.s }[0], [x20], #0x4\n"
1949 "tbz x1, #1, 104f\n"
1950 "ld1 { v2.h }[2], [x20], #0x2\n"
1951 "tbz x1, #0, 107f\n"
1952 "ld1 { v2.b }[6], [x20]\n"
1955 "tbz x1, #0, 107f\n"
1956 "ld1 { v2.b }[4], [x20]\n"
1959 "tbz x1, #1, 106f\n"
1960 "ld1 { v2.h }[0], [x20], #0x2\n"
1961 "tbz x1, #0, 107f\n"
1962 "ld1 { v2.b }[2], [x20]\n"
1965 "tbz x1, #0, 107f\n"
1966 "ld1 { v2.b }[0], [x20]\n"
1968 "ldr d30, [x6, #0xb8]\n"
1969 "usubl v2.8h, v2.8b, v18.8b\n"
1970 "ssubl v30.8h, v30.8b, v13.8b\n"
1971 "ldr x20, [x5, #0x110]\n"
1972 "smlal v23.4s, v2.4h, v28.4h\n"
1973 "smlal2 v19.4s, v2.8h, v28.8h\n"
1974 "add x20, x20, x3\n"
1975 "smlal v7.4s, v29.4h, v30.4h\n"
1976 "smlal2 v15.4s, v29.8h, v30.8h\n"
1977 "smlal v20.4s, v25.4h, v30.4h\n"
1978 "smlal2 v5.4s, v25.8h, v30.8h\n"
1979 "smlal v24.4s, v2.4h, v30.4h\n"
1980 "smlal2 v22.4s, v2.8h, v30.8h\n"
1981 "tbz x1, #2, 109f\n"
1982 "ld1 { v27.s }[0], [x20], #0x4\n"
1983 "tbz x1, #1, 108f\n"
1984 "ld1 { v27.h }[2], [x20], #0x2\n"
1985 "tbz x1, #0, 111f\n"
1986 "ld1 { v27.b }[6], [x20]\n"
1989 "tbz x1, #0, 111f\n"
1990 "ld1 { v27.b }[4], [x20]\n"
1993 "tbz x1, #1, 110f\n"
1994 "ld1 { v27.h }[0], [x20], #0x2\n"
1995 "tbz x1, #0, 111f\n"
1996 "ld1 { v27.b }[2], [x20]\n"
1999 "tbz x1, #0, 111f\n"
2000 "ld1 { v27.b }[0], [x20]\n"
2002 "ldr d8, [x6, #0xc0]\n"
2003 "usubl v27.8h, v27.8b, v18.8b\n"
2004 "ssubl v8.8h, v8.8b, v13.8b\n"
2005 "ldr x20, [x5, #0x118]\n"
2006 "smlal v23.4s, v27.4h, v30.4h\n"
2007 "smlal2 v19.4s, v27.8h, v30.8h\n"
2008 "add x20, x20, x3\n"
2009 "smlal v7.4s, v25.4h, v8.4h\n"
2010 "smlal2 v15.4s, v25.8h, v8.8h\n"
2011 "smlal v20.4s, v21.4h, v8.4h\n"
2012 "smlal2 v5.4s, v21.8h, v8.8h\n"
2013 "smlal v24.4s, v27.4h, v8.4h\n"
2014 "smlal2 v22.4s, v27.8h, v8.8h\n"
2015 "tbz x1, #2, 113f\n"
2016 "ld1 { v9.s }[0], [x20], #0x4\n"
2017 "tbz x1, #1, 112f\n"
2018 "ld1 { v9.h }[2], [x20], #0x2\n"
2019 "tbz x1, #0, 115f\n"
2020 "ld1 { v9.b }[6], [x20]\n"
2023 "tbz x1, #0, 115f\n"
2024 "ld1 { v9.b }[4], [x20]\n"
2027 "tbz x1, #1, 114f\n"
2028 "ld1 { v9.h }[0], [x20], #0x2\n"
2029 "tbz x1, #0, 115f\n"
2030 "ld1 { v9.b }[2], [x20]\n"
2033 "tbz x1, #0, 115f\n"
2034 "ld1 { v9.b }[0], [x20]\n"
2036 "usubl v9.8h, v9.8b, v18.8b\n"
2037 "smlal v23.4s, v9.4h, v8.4h\n"
2038 "smlal2 v19.4s, v9.8h, v8.8h\n"
2039 "tbz x1, #2, 117f\n"
2040 "ld1 { v30.4s }, [x7], #0x10\n"
2041 "ld1 { v12.4s }, [x8], #0x10\n"
2042 "tbz x1, #1, 116f\n"
2043 "ld1 { v14.d }[0], [x7], #0x8\n"
2044 "ld1 { v27.d }[0], [x8], #0x8\n"
2045 "tbz x1, #0, 119f\n"
2046 "ld1 { v14.s }[2], [x7]\n"
2047 "ld1 { v27.s }[2], [x8]\n"
2050 "tbz x1, #0, 119f\n"
2051 "ld1 { v14.s }[0], [x7]\n"
2052 "ld1 { v27.s }[0], [x8]\n"
2055 "tbz x1, #1, 118f\n"
2056 "ld1 { v30.d }[0], [x7], #0x8\n"
2057 "ld1 { v12.d }[0], [x8], #0x8\n"
2058 "tbz x1, #0, 119f\n"
2059 "ld1 { v30.s }[2], [x7]\n"
2060 "ld1 { v12.s }[2], [x8]\n"
2063 "tbz x1, #0, 119f\n"
2064 "ld1 { v30.s }[0], [x7]\n"
2065 "ld1 { v12.s }[0], [x8]\n"
2067 "sqrdmulh v7.4s, v7.4s, v30.4s\n"
2068 "and v16.16b, v7.16b, v12.16b\n"
2069 "add x17, x17, x4\n"
2070 "add x16, x16, x4\n"
2071 "sqrdmulh v15.4s, v15.4s, v14.4s\n"
2072 "sshr v16.4s, v16.4s, #0x1f\n"
2073 "add x15, x15, x4\n"
2074 "add x14, x14, x4\n"
2075 "and v2.16b, v15.16b, v27.16b\n"
2076 "sqrdmulh v20.4s, v20.4s, v30.4s\n"
2077 "sqrdmulh v24.4s, v24.4s, v30.4s\n"
2078 "sqrdmulh v23.4s, v23.4s, v30.4s\n"
2079 "sqadd v7.4s, v7.4s, v16.4s\n"
2080 "sshr v2.4s, v2.4s, #0x1f\n"
2081 "and v21.16b, v20.16b, v12.16b\n"
2082 "sqrdmulh v5.4s, v5.4s, v14.4s\n"
2083 "and v18.16b, v24.16b, v12.16b\n"
2084 "sqrdmulh v22.4s, v22.4s, v14.4s\n"
2085 "and v31.16b, v23.16b, v12.16b\n"
2086 "sqrdmulh v19.4s, v19.4s, v14.4s\n"
2087 "sqadd v15.4s, v15.4s, v2.4s\n"
2088 "sshr v21.4s, v21.4s, #0x1f\n"
2089 "and v9.16b, v5.16b, v27.16b\n"
2090 "sshr v18.4s, v18.4s, #0x1f\n"
2091 "and v4.16b, v22.16b, v27.16b\n"
2092 "sshr v31.4s, v31.4s, #0x1f\n"
2093 "and v28.16b, v19.16b, v27.16b\n"
2094 "sqadd v20.4s, v20.4s, v21.4s\n"
2095 "sshr v9.4s, v9.4s, #0x1f\n"
2096 "sqadd v24.4s, v24.4s, v18.4s\n"
2097 "sshr v4.4s, v4.4s, #0x1f\n"
2098 "sqadd v23.4s, v23.4s, v31.4s\n"
2099 "sshr v28.4s, v28.4s, #0x1f\n"
2100 "srshl v7.4s, v7.4s, v12.4s\n"
2101 "srshl v20.4s, v20.4s, v12.4s\n"
2102 "sqadd v5.4s, v5.4s, v9.4s\n"
2103 "srshl v24.4s, v24.4s, v12.4s\n"
2104 "sqadd v22.4s, v22.4s, v4.4s\n"
2105 "srshl v23.4s, v23.4s, v12.4s\n"
2106 "sqadd v19.4s, v19.4s, v28.4s\n"
2107 "srshl v15.4s, v15.4s, v27.4s\n"
2108 "sqxtn v7.4h, v7.4s\n"
2109 "srshl v5.4s, v5.4s, v27.4s\n"
2110 "sqxtn v20.4h, v20.4s\n"
2111 "srshl v22.4s, v22.4s, v27.4s\n"
2112 "sqxtn v24.4h, v24.4s\n"
2113 "srshl v19.4s, v19.4s, v27.4s\n"
2114 "sqxtn v23.4h, v23.4s\n"
2115 "sqxtn2 v7.8h, v15.4s\n"
2116 "sqxtn2 v20.8h, v5.4s\n"
2117 "sqxtn2 v24.8h, v22.4s\n"
2118 "sqxtn2 v23.8h, v19.4s\n"
2119 "sqadd v7.8h, v7.8h, v26.8h\n"
2120 "sqadd v20.8h, v20.8h, v26.8h\n"
2121 "sqadd v24.8h, v24.8h, v26.8h\n"
2122 "sqadd v23.8h, v23.8h, v26.8h\n"
2123 "smax v7.8h, v7.8h, v11.8h\n"
2124 "smax v20.8h, v20.8h, v11.8h\n"
2125 "smax v24.8h, v24.8h, v11.8h\n"
2126 "smax v23.8h, v23.8h, v11.8h\n"
2127 "smin v7.8h, v7.8h, v0.8h\n"
2128 "smin v20.8h, v20.8h, v0.8h\n"
2129 "smin v24.8h, v24.8h, v0.8h\n"
2130 "smin v23.8h, v23.8h, v0.8h\n"
2131 "uzp1 v7.16b, v7.16b, v7.16b\n"
2132 "uzp1 v20.16b, v20.16b, v20.16b\n"
2133 "uzp1 v24.16b, v24.16b, v24.16b\n"
2134 "uzp1 v23.16b, v23.16b, v23.16b\n"
2135 "tbz x1, #2, 121f\n"
2136 "st1 { v7.s }[0], [x17], #0x4\n"
2137 "st1 { v20.s }[0], [x16], #0x4\n"
2138 "st1 { v24.s }[0], [x15], #0x4\n"
2139 "st1 { v23.s }[0], [x14], #0x4\n"
2140 "tbz x1, #1, 120f\n"
2141 "st1 { v7.h }[2], [x17], #0x2\n"
2142 "st1 { v20.h }[2], [x16], #0x2\n"
2143 "st1 { v24.h }[2], [x15], #0x2\n"
2144 "st1 { v23.h }[2], [x14], #0x2\n"
2145 "tbz x1, #0, 123f\n"
2146 "st1 { v7.b }[6], [x17], #0x1\n"
2147 "st1 { v20.b }[6], [x16], #0x1\n"
2148 "st1 { v24.b }[6], [x15], #0x1\n"
2149 "st1 { v23.b }[6], [x14], #0x1\n"
2152 "tbz x1, #0, 123f\n"
2153 "st1 { v7.b }[4], [x17], #0x1\n"
2154 "st1 { v20.b }[4], [x16], #0x1\n"
2155 "st1 { v24.b }[4], [x15], #0x1\n"
2156 "st1 { v23.b }[4], [x14], #0x1\n"
2159 "tbz x1, #1, 122f\n"
2160 "st1 { v7.h }[0], [x17], #0x2\n"
2161 "st1 { v20.h }[0], [x16], #0x2\n"
2162 "st1 { v24.h }[0], [x15], #0x2\n"
2163 "st1 { v23.h }[0], [x14], #0x2\n"
2164 "tbz x1, #0, 123f\n"
2165 "st1 { v7.b }[2], [x17], #0x1\n"
2166 "st1 { v20.b }[2], [x16], #0x1\n"
2167 "st1 { v24.b }[2], [x15], #0x1\n"
2168 "st1 { v23.b }[2], [x14], #0x1\n"
2171 "tbz x1, #0, 123f\n"
2172 "st1 { v7.b }[0], [x17], #0x1\n"
2173 "st1 { v20.b }[0], [x16], #0x1\n"
2174 "st1 { v24.b }[0], [x15], #0x1\n"
2175 "st1 { v23.b }[0], [x14], #0x1\n"
2179 : [offsetof_Params_bias]
"I" (offsetof(Params,
bias)), [offsetof_Params_inptrs]
"I" (offsetof(Params, inptrs)), [offsetof_Params_n_channels]
"I" (offsetof(Params, n_channels)), [offsetof_Params_outptrs]
"I" (offsetof(Params, outptrs)), [offsetof_Params_requant]
"I" (offsetof(Params, requant)), [offsetof_Params_requant_muls]
"I" (offsetof(Params,
requant_muls)), [offsetof_Params_requant_shifts]
"I" (offsetof(Params,
requant_shifts)), [offsetof_Params_weights]
"I" (offsetof(Params, weights)), [offsetof_Requantize32_a_offset]
"I" (offsetof(
arm_gemm::Requantize32, a_offset)), [offsetof_Requantize32_b_offset]
"I" (offsetof(
arm_gemm::Requantize32, b_offset)), [offsetof_Requantize32_c_offset]
"I" (offsetof(
arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval]
"I" (offsetof(
arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval]
"I" (offsetof(
arm_gemm::Requantize32, minval)), [params]
"r" (¶ms)
2180 :
"cc",
"memory",
"v0",
"v1",
"v2",
"v3",
"v4",
"v5",
"v6",
"v7",
"v8",
"v9",
"v10",
"v11",
"v12",
"v13",
"v14",
"v15",
"v16",
"v17",
"v18",
"v19",
"v20",
"v21",
"v22",
"v23",
"v24",
"v25",
"v26",
"v27",
"v28",
"v29",
"v30",
"v31",
"x1",
"x2",
"x3",
"x4",
"x5",
"x6",
"x7",
"x8",
"x9",
"x10",
"x11",
"x12",
"x13",
"x14",
"x15",
"x16",
"x17",
"x20",
"x21",
"x22",
"x23",
"x24",
"x25",
"x26",
"x27",
"x28"
2187 #endif // defined(__aarch64__)