45 void initialize_matrix_transform(SimpleTensor<T> &
src,
const Size2D &output_tile_size,
const Size2D &kernel_size,
WinogradTransformType winograd_transform_type)
48 static const std::array<float, 16> imatrix2x2_3x3 =
50 1.0f, 0.0f, -1.0f, 0.0f,
51 0.0f, 1.0f, 1.0f, 0.0f,
52 0.0f, -1.0f, 1.0f, 0.0f,
53 0.0f, 1.0f, 0.0f, -1.0f
56 static const std::array<float, 36> imatrix4x4_3x3 =
58 4.0f, 0.0f, -5.0f, 0.0f, 1.0f, 0.0f,
59 0.0f, -4.0f, -4.0f, 1.0f, 1.0f, 0.0f,
60 0.0f, 4.0f, -4.0f, -1.0f, 1.0f, 0.0f,
61 0.0f, -2.0f, -1.0f, 2.0f, 1.0f, 0.0f,
62 0.0f, 2.0f, -1.0f, -2.0f, 1.0f, 0.0f,
63 0.0f, 4.0f, 0.0f, -5.0f, 0.0f, 1.0f,
66 static const std::array<float, 64> imatrix4x4_5x5 =
68 1.f, 0.f, -21.f / 4.f, 0.f, 21.f / 4.f, 0.f, -1.f, 0.f,
69 0.f, 1.f, 1.f, -17.f / 4.f, -17.f / 4.f, 1.f, 1.f, 0.f,
70 0.f, -1.f, 1.f, 17.f / 4.f, -17.f / 4.f, -1.f, 1.f, 0.f,
71 0.f, 1.f / 2.f, 1.f / 4.f, -5.f / 2.f, -5.f / 4.f, 2.f, 1.f, 0.f,
72 0.f, -1.f / 2.f, 1.f / 4.f, 5.f / 2.f, -5.f / 4.f, -2.f, 1.f, 0.f,
73 0.f, 2.f, 4.f, -5.f / 2.f, -5.f, 1.f / 2.f, 1.f, 0.f,
74 0.f, -2.f, 4.f, 5.f / 2.f, -5.f, -1.f / 2.f, 1.f, 0.f,
75 0.f, -1.f, 0.f, 21.f / 4.f, 0.f, -21.f / 4.f, 0.f, 1.f
78 static const std::array<float, 64> imatrix2x1_7x7 =
80 -36.0f, 0.0f, 49.0f, 0.0f, -14.0f, 0.0f, 1.0f, 0.0f,
81 0.0f, -36.0f, 36.0f, 13.0f, -13.0f, -1.0f, 1.0f, 0.0f,
82 0.0f, 36.0f, 36.0f, -13.0f, -13.0f, 1.0f, 1.0f, 0.0f,
83 0.0f, -18.0f, 9.0f, 20.0f, -10.0f, -2.0f, 1.0f, 0.0f,
84 0.0f, 18.0f, 9.0f, -20.0f, -10.0f, 2.0f, 1.0f, 0.0f,
85 0.0f, -12.0f, 4.0f, 15.0f, -5.0f, -3.0f, 1.0f, 0.0f,
86 0.0f, 12.0f, 4.0f, -15.0f, -5.0f, 3.0f, 1.0f, 0.0f,
87 0.0f, -36.0f, 0.0f, 49.0f, 0.0f, -14.0f, 0.0f, 1.0f
93 static const std::array<float, 12> fmatrix2x2_3x3 =
101 static const std::array<float, 18> fmatrix4x4_3x3 =
104 -1.0f / 6.0f, -1.0f / 6.0f, -1.0f / 6.0f,
105 -1.0f / 6.0f, 1.0f / 6.0f, -1.0f / 6.0f,
106 1.0f / 24.0f, 1.0f / 12.0f, 1.0f / 6.0f,
107 1.0f / 24.0f, -1.0f / 12.0f, 1.0f / 6.0f,
111 static const std::array<float, 40> fmatrix4x4_5x5 =
113 1.0f, 0.0f, 0.0f, 0.0f, 0.0f,
114 -2.0f / 9.0f, -2.0f / 9.0f, -2.0f / 9.0f, -2.0f / 9.0f, -2.0f / 9.0f,
115 -2.0f / 9.0f, 2.0f / 9.0f, -2.0f / 9.0f, 2.0f / 9.0f, -2.0f / 9.0f,
116 1.0f / 90.0f, 1.0f / 45.0f, 2.0f / 45.0f, 4.0f / 45.0f, 8.0f / 45.0f,
117 1.0f / 90.0f, -1.0f / 45.0f, 2.0f / 45.0f, -4.0f / 45.0f, 8.0f / 45.0f,
118 4.0f / 45.0f, 2.0f / 45.0f, 1.0f / 45.0f, 1.0f / 90.0f, 1.0f / 180.0f,
119 4.0f / 45.0f, -2.0f / 45.0f, 1.0f / 45.0f, -1.0f / 90.0f, 1.0f / 180.0f,
120 0.0f, 0.0f, 0.0f, 0.0f, 1.0f
124 static const std::array<float, 56> fmatrix2x1_7x7 =
126 -1.0f / 36.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f,
127 1.0f / 48.0f, -1.0f / 48.0f, 1.0f / 48.0f, -1.0f / 48.0f, 1.0f / 48.0f, -1.0f / 48.0f, 1.0f / 48.0f,
128 1.0f / 48.0f, 1.0f / 48.0f, 1.0f / 48.0f, 1.0f / 48.0f, 1.0f / 48.0f, 1.0f / 48.0f, 1.0f / 48.0f,
129 -1.0f / 120.0f, 1.0f / 60.0f, -1.0f / 30.0f, 1.0f / 15.0f, -2.0f / 15.0f, 4.0f / 15.0f, -8.0f / 15.0f,
130 -1.0f / 120.0f, -1.0f / 60.0f, -1.0f / 30.0f, -1.0f / 15.0f, -2.0f / 15.0f, -4.0f / 15.0f, -8.0f / 15.0f,
131 1.0f / 720.0f, -1.0f / 240.0f, 1.0f / 80.0f, -3.0f / 80.0f, 9.0f / 80.0f, -27.0f / 80.0f, 81.0f / 80.0f,
132 1.0f / 720.0f, 1.0f / 240.0f, 1.0f / 80.0f, 3.0f / 80.0f, 9.0f / 80.0f, 27.0f / 80.0f, 81.0f / 80.0f,
133 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 1.0f
139 static const std::array<float, 8> omatrix2x2_3x3 =
141 1.0f, 1.0f, 1.0f, 0.0f,
142 0.0f, 1.0f, -1.0f, -1.0f
145 static const std::array<float, 24> omatrix4x4_3x3 =
147 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 0.0f,
148 0.0f, 1.0f, -1.0f, 2.0f, -2.0f, 0.0f,
149 0.0f, 1.0f, 1.0f, 4.0f, 4.0f, 0.0f,
150 0.0f, 1.0f, -1.0f, 8.0f, -8.0f, 1.0f
153 static const std::array<float, 36> omatrix4x4_5x5 =
155 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 8.0f, 8.0f, 0.0f,
156 0.0f, 1.0f, -1.0f, 2.0f, -2.0f, 4.0f, -4.0f, 0.0f,
157 0.0f, 1.0f, 1.0f, 4.0f, 4.0f, 2.0f, 2.0f, 0.0f,
158 0.0f, 1.0f, -1.0f, 8.0f, -8.0f, 1.0f, -1.0f, 1.0f
161 static const std::array<float, 16> omatrix2x1_7x7 =
163 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 0.0f,
164 0.0f, -1.0f, 1.0f, -2.0f, 2.0f, -3.0f, 3.0f, 1.0f
172 static std::map<WinogradKey, const float *> matrix_map =
213 std::map<WinogradKey, const float *>::iterator it;
215 it = matrix_map.find(WinogradKey(std::pair<int, int>(output_tile_size.width, output_tile_size.height),
216 std::pair<int, int>(kernel_size.width, kernel_size.height),
217 winograd_transform_type));
219 float const *matrix_values =
nullptr;
220 if(it != matrix_map.end())
223 matrix_values = it->second;
231 std::copy(&matrix_values[0], &matrix_values[0] +
src.num_elements(), &
src[0]);
235 template <
typename T>
247 const unsigned int tile_w = output_tile_size.
width + kernel_size.
width - 1;
248 const unsigned int tile_h = output_tile_size.
height + kernel_size.
height - 1;
251 const unsigned int tile_max_dim = std::max(tile_w, tile_h);
274 transpose_matrix<T>(matrix, matrix_transposed);
276 const int in_w = in.
shape().x();
277 const int in_h = in.
shape().y();
278 const int in_d = in.
shape().z();
279 const int out_d = out.shape().z();
280 const int num_batches = in.
shape().total_size() / (in_w * in_h * in_d);
281 const int step_x = output_tile_size.
width;
282 const int step_y = output_tile_size.
height;
290 const int num_tiles_x = num_tiles.
width;
291 const int num_tiles_y = num_tiles.
height;
294 int start_x_zero = 0;
295 int start_y_zero = 0;
299 if(output_tile_size.
width == 1)
303 end_x_zero = tile_max_dim - 1;
304 end_y_zero = tile_max_dim;
306 else if(output_tile_size.
height == 1)
310 end_x_zero = tile_max_dim;
311 end_y_zero = tile_max_dim - 1;
315 const Coordinates anchor_zeros(start_x_zero, start_y_zero);
316 const TensorShape shape_zeros(end_x_zero, end_y_zero);
319 const int step_y_transf_tile = kernel_size.
width == 1 ? tile_max_dim : 1;
323 for(
int b = 0;
b < num_batches; ++
b)
325 for(
int z = 0; z < in_d; ++z)
327 for(
int y = 0; y < num_tiles_y; ++y)
329 for(
int x = 0; x < num_tiles_x; ++x)
331 int xi = x * step_x -
conv_info.pad_left();
332 int yi = y * step_y -
conv_info.pad_top();
338 zeros<T>(src_tile, anchor_zeros, shape_zeros);
341 matrix_multiply<T>(matrix, src_tile, tmp_tile);
342 matrix_multiply<T>(tmp_tile, matrix_transposed, dst_tile);
345 for(
int i = 0; i < out_d; ++i)
348 int yo = x + y * num_tiles_x;
359 template <
typename T>
371 const unsigned int input_tile_w = output_tile_size.
width + kernel_size.
width - 1;
372 const unsigned int input_tile_h = output_tile_size.
height + kernel_size.
height - 1;
373 const unsigned int input_tile_area = input_tile_w * input_tile_h;
376 const unsigned int kernel_max_dim = std::max(kernel_size.
width, kernel_size.
height);
379 const unsigned int input_tile_max_dim = std::max(input_tile_w, input_tile_h);
400 transpose_matrix<T>(trans_matrix, trans_matrix_transposed);
402 const int num_channels = in.
shape()[2];
403 const int num_filters = in.
shape()[3];
404 const int num_batches = in.
shape().total_size() / (kernel_size.
area() * num_channels * num_filters);
407 const int step_y_transf_tile = kernel_size.
width == 1 ? input_tile_max_dim : 1;
409 for(
int n = 0; n < num_batches; ++n)
411 for(
int w = 0;
w < num_filters; ++
w)
413 for(
int z = 0; z < num_channels; ++z)
416 get_tile<T>(in, input_tile,
Coordinates(0, 0, z,
w, n));
419 matrix_multiply<T>(trans_matrix, input_tile, tmp_tile);
422 matrix_multiply<T>(tmp_tile, trans_matrix_transposed, transf_tile);
425 const int output_offset =
w + z * num_filters;
428 for(
unsigned int i = 0; i < input_tile_area; ++i)
430 out[output_offset + i * num_filters * num_channels] = transf_tile[i * step_y_transf_tile];
439 template <
typename T>
451 const unsigned int in_tile_w = output_tile_size.
width + kernel_size.
width - 1;
452 const unsigned int in_tile_h = output_tile_size.
height + kernel_size.
height - 1;
453 const unsigned int out_tile_w = output_tile_size.
width;
454 const unsigned int out_tile_h = output_tile_size.
height;
460 const unsigned int in_tile_max_dim = std::max(in_tile_w, in_tile_h);
461 const unsigned int out_tile_max_dim = std::max(output_tile_size.
width, output_tile_size.
height);
465 TensorShape in_tile_dims(in_tile_max_dim, in_tile_max_dim);
468 TensorShape out_tile_dims(out_tile_max_dim, out_tile_max_dim);
471 TensorShape tr_tile_dims(in_tile_max_dim, out_tile_max_dim);
493 transpose_matrix<T>(trans_matrix, trans_matrix_transposed);
495 const int w_in = in.
shape()[0];
496 const int h_in = in.
shape()[1];
497 const int c_in = in.
shape()[2];
498 const int w_out = out.shape()[0];
499 const int h_out = out.shape()[1];
500 const int c_out = out.shape()[2];
501 const int num_batches = in.
shape().total_size() / (w_in * h_in * c_in);
504 const int stridey_in = w_in;
505 const int stridez_in = stridey_in * h_in;
506 const int stridew_in = stridez_in * c_in;
509 const int stridey_out = w_out;
510 const int stridez_out = stridey_out * h_out;
511 const int stridew_out = stridez_out * c_out;
519 const int num_tiles_x = num_tiles.
width;
520 const int num_tiles_y = num_tiles.
height;
526 const int step_y_transf_tile = kernel_size.
width == 1 ? 1 : output_tile.shape()[0];
529 zeros<T>(input_tile,
Coordinates(0, 0), input_tile.shape());
531 for(
int n = 0; n < num_batches; ++n)
533 for(
int y = 0; y < h_in; ++y)
535 for(
int x = 0; x < w_in; ++x)
538 for(
int z = 0; z < c_in; ++z)
540 input_tile[z] = in[x + (y * stridey_in) + (z * stridez_in) + (n * stridew_in)];
544 matrix_multiply<T>(trans_matrix, input_tile, tmp_tile);
547 matrix_multiply<T>(tmp_tile, trans_matrix_transposed, output_tile);
550 const int xo = (y % num_tiles_x) * out_tile_w;
551 const int yo = (y / num_tiles_x) * out_tile_h;
554 const int output_offset = xo + (yo * stridey_out) + (zo * stridez_out) + (n * stridew_out);
556 for(
int yi = 0; yi < static_cast<int>(out_tile_h); ++yi)
558 for(
int xi = 0; xi < static_cast<int>(out_tile_w); ++xi)
561 if((xo + xi < w_out) && (yo + yi < h_out))
563 out[output_offset + yi * stridey_out + xi] = output_tile[xi + yi * step_y_transf_tile];
566 out[output_offset + yi * stridey_out + xi] +=
b[zo];