25 #if defined(ARM_COMPUTE_ENABLE_SVE)
29 #include "depthwise.hpp"
35 struct interleave_sve_s8q_3x3_dot
37 static size_t get_packed_size(
const DepthwiseArgs &);
41 size_t interleave_sve_s8q_3x3_dot::get_packed_size(
const DepthwiseArgs &
args)
54 "cmp %x[ld_weight_col], XZR\n"
55 "csel %x[ld_weight_col], %x[ld_weight_col], %x[n_channels], NE\n"
60 "mul x20, %x[ld_weight_col], x20\n"
61 "ld1rw { z27.s }, p2/Z, [%x[qp], %[offsetof_input_offset]]\n"
62 "ld1rw { z26.s }, p2/Z, [%x[qp], %[offsetof_weights_offset]]\n"
63 "cmp %x[ld_weight_row], XZR\n"
64 "csel %x[ld_weight_row], %x[ld_weight_row], x20, NE\n"
66 "mul z26.s, p2/M, z26.s, z27.s\n"
67 "add x24, %x[weights], %x[ld_weight_row]\n"
68 "ld1rw { z24.s }, p2/Z, [%x[qp], %[offsetof_per_layer_mul]]\n"
69 "ld1rw { z23.s }, p2/Z, [%x[qp], %[offsetof_per_layer_right_shift]]\n"
70 "add x23, x24, %x[ld_weight_row]\n"
71 "add x22, %x[ld_weight_col], %x[ld_weight_col]\n"
72 "whilelt p1.s, XZR, %x[n_channels]\n"
74 "mul z26.s, p2/M, z26.s, z16.s\n"
80 "cntp x20, p2, p1.s\n"
81 "whilelt p0.b, XZR, x20\n"
82 "ld1b { z18.b }, p0/Z, [%x[weights]]\n"
83 "ld1b { z17.b }, p0/Z, [%x[weights], %x[ld_weight_col]]\n"
84 "ld1b { z16.b }, p0/Z, [%x[weights], x22]\n"
85 "zip1 z20.b, z18.b, z16.b\n"
86 "zip1 z19.b, z17.b, z28.b\n"
87 "ld1b { z18.b }, p0/Z, [x24]\n"
88 "ld1b { z17.b }, p0/Z, [x24, %x[ld_weight_col]]\n"
89 "ld1b { z16.b }, p0/Z, [x24, x22]\n"
90 "zip1 z22.b, z20.b, z19.b\n"
91 "zip1 z21.b, z18.b, z16.b\n"
92 "zip1 z19.b, z17.b, z28.b\n"
94 "ld1b { z18.b }, p0/Z, [x23]\n"
95 "ld1b { z17.b }, p0/Z, [x23, %x[ld_weight_col]]\n"
96 "ld1b { z16.b }, p0/Z, [x23, x22]\n"
97 "sdot z20.s, z25.b, z22.b\n"
98 "zip1 z19.b, z21.b, z19.b\n"
99 "sdot z20.s, z25.b, z19.b\n"
100 "zip1 z18.b, z18.b, z16.b\n"
101 "zip1 z16.b, z17.b, z28.b\n"
102 "and p0.b, p2/Z, p8.b, p1.b\n"
103 "ld1w { z17.s }, p0/Z, [%x[bias], x21, LSL #2]\n"
104 "zip1 z16.b, z18.b, z16.b\n"
105 "sdot z20.s, z25.b, z16.b\n"
106 "mls z17.s, p2/M, z20.s, z27.s\n"
107 "add %x[weights], %x[weights], x20\n"
108 "add x24, x24, x20\n"
109 "add x23, x23, x20\n"
110 "add z17.s, z17.s, z26.s\n"
111 "st1w { z17.s }, p2, [%x[outptr]]\n"
112 "st1b { z22.b }, p2, [%x[outptr], #1, MUL VL]\n"
113 "st1b { z19.b }, p2, [%x[outptr], #2, MUL VL]\n"
114 "st1b { z16.b }, p2, [%x[outptr], #3, MUL VL]\n"
115 "addvl %x[outptr], %x[outptr], #4\n"
116 "cbz %x[rq_mul_perchannel], 3f\n"
117 "ld1w { z24.s }, p1/Z, [%x[rq_mul_perchannel], x21, LSL #2]\n"
118 "ld1w { z23.s }, p1/Z, [%x[rq_shift_perchannel], x21, LSL #2]\n"
121 "whilelt p1.s, x21, %x[n_channels]\n"
122 "st1w { z24.s }, p2, [%x[outptr]]\n"
123 "st1w { z23.s }, p2, [%x[outptr], #1, MUL VL]\n"
124 "addvl %x[outptr], %x[outptr], #2\n"
126 : [ld_weight_col]
"+&r" (ld_weight_col), [ld_weight_row]
"+&r" (ld_weight_row), [outptr]
"+&r" (outptr), [weights]
"+&r" (weights)
127 : [
bias]
"r" (
bias), [n_channels]
"r" (n_channels), [offsetof_input_offset]
"I" (offsetof(
arm_gemm::Requantize32, a_offset)), [offsetof_per_layer_mul]
"I" (offsetof(
arm_gemm::Requantize32, per_layer_mul)), [offsetof_per_layer_right_shift]
"I" (offsetof(
arm_gemm::Requantize32, per_layer_right_shift)), [offsetof_weights_offset]
"I" (offsetof(
arm_gemm::Requantize32, b_offset)), [qp]
"r" (&qp), [rq_mul_perchannel]
"r" (qp.
per_channel_muls), [rq_shift_perchannel]
"r" (qp.
per_channel_right_shifts)
128 :
"cc",
"memory",
"p0",
"p1",
"p2",
"p8",
"x20",
"x21",
"x22",
"x23",
"x24",
"z16",
"z17",
"z18",
"z19",
"z20",
"z21",
"z22",
"z23",
"z24",
"z25",
"z26",
"z27",
"z28"
135 #endif // defined(ARM_COMPUTE_ENABLE_SVE)