Compute Library
 21.02
neon_opticalflow.cpp
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2019 Arm Limited.
3  *
4  * SPDX-License-Identifier: MIT
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a copy
7  * of this software and associated documentation files (the "Software"), to
8  * deal in the Software without restriction, including without limitation the
9  * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10  * sell copies of the Software, and to permit persons to whom the Software is
11  * furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in all
14  * copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22  * SOFTWARE.
23  */
25 
26 #include "arm_compute/core/Types.h"
27 #include "utils/ImageLoader.h"
28 #include "utils/Utils.h"
29 
30 #include <fstream>
31 #include <sstream>
32 #include <vector>
33 
34 using namespace arm_compute;
35 using namespace utils;
36 
37 class NeonOpticalFlowExample : public Example
38 {
39 public:
40  NeonOpticalFlowExample()
41  : input_points(100), output_points(100), point_estimates(100)
42  {
43  }
44 
45  bool do_setup(int argc, char **argv) override
46  {
47  if(argc < 5)
48  {
49  // Print help
50  std::cout << "Usage: ./build/neon_opticalflow [src_1st.ppm] [src_2nd.ppm] [keypoints] [estimates]\n\n";
51  const unsigned int img_width = 64;
52  const unsigned int img_height = 64;
53  const unsigned int rect_x = 20;
54  const unsigned int rect_y = 40;
55  const unsigned int rect_s = 8;
56  const unsigned int offsetx = 24;
57  const unsigned int offsety = 3;
58  std::cout << "No input_image provided, creating test data:\n";
59  std::cout << "\t Image src_1st = (" << img_width << "," << img_height << ")" << std::endl;
60  std::cout << "\t Image src_2nd = (" << img_width << "," << img_height << ")" << std::endl;
61  init_img(src_1st, img_width, img_height, rect_x, rect_y, rect_s);
62  init_img(src_2nd, img_width, img_height, rect_x + offsetx, rect_y + offsety, rect_s);
63  const int num_points = 4;
64  input_points.resize(num_points);
65  point_estimates.resize(num_points);
66  const std::array<unsigned int, num_points> tracking_coordsx = { rect_x - 1, rect_x, rect_x + 1, rect_x + 2 };
67  const std::array<unsigned int, num_points> tracking_coordsy = { rect_y - 1, rect_y, rect_y + 1, rect_y + 2 };
68  const std::array<unsigned int, num_points> estimate_coordsx = { rect_x + offsetx - 1, rect_x + offsetx, rect_x + offsetx + 1, rect_x + offsetx + 2 };
69  const std::array<unsigned int, num_points> estimate_coordsy = { rect_y + offsety - 1, rect_y + offsety, rect_y + offsety + 1, rect_y + offsety + 2 };
70 
71  for(int k = 0; k < num_points; ++k)
72  {
73  auto &keypoint = input_points.at(k);
74  keypoint.x = tracking_coordsx[k];
75  keypoint.y = tracking_coordsy[k];
76  keypoint.tracking_status = 1;
77  }
78  for(int k = 0; k < num_points; ++k)
79  {
80  auto &keypoint = point_estimates.at(k);
81  keypoint.x = estimate_coordsx[k];
82  keypoint.y = estimate_coordsy[k];
83  keypoint.tracking_status = 1;
84  }
85  }
86  else
87  {
88  load_ppm(argv[1], src_1st);
89  load_ppm(argv[2], src_2nd);
90  load_keypoints(argv[3], input_points);
91  load_keypoints(argv[4], point_estimates);
92  }
93 
94  print_points(input_points, "Tracking points : ");
95  print_points(point_estimates, "Estimates points : ");
96 
97  const unsigned int num_levels = 3;
98  // Initialise and allocate pyramids
99  PyramidInfo pyramid_info(num_levels, SCALE_PYRAMID_HALF, src_1st.info()->tensor_shape(), src_1st.info()->format());
100  pyr_1st.init_auto_padding(pyramid_info);
101  pyr_2nd.init_auto_padding(pyramid_info);
102 
103  pyrf_1st.configure(&src_1st, &pyr_1st, BorderMode::UNDEFINED, 0);
104  pyrf_2nd.configure(&src_2nd, &pyr_2nd, BorderMode::UNDEFINED, 0);
105 
106  output_points.resize(input_points.num_values());
107 
108  optkf.configure(&pyr_1st, &pyr_2nd,
109  &input_points, &point_estimates, &output_points,
111 
112  pyr_1st.allocate();
113  pyr_2nd.allocate();
114 
115  return true;
116  }
117  void do_run() override
118  {
119  //Execute the functions:
120  pyrf_1st.run();
121  pyrf_2nd.run();
122  optkf.run();
123  }
124  void do_teardown() override
125  {
126  print_points(output_points, "Output points : ");
127  }
128 
129 private:
130  /** Loads the input keypoints from a file into an array
131  *
132  * @param[in] fn Filename containing the keypoints. Each line must have two values X Y.
133  * @param[out] img Reference to an unintialised KeyPointArray
134  */
135  bool load_keypoints(const std::string &fn, KeyPointArray &array)
136  {
137  assert(!fn.empty());
138  std::ifstream f(fn);
139  if(f.is_open())
140  {
141  std::cout << "Reading points from " << fn << std::endl;
142  std::vector<KeyPoint> v;
143  for(std::string line; std::getline(f, line);)
144  {
145  std::stringstream ss(line);
146  std::string xcoord;
147  std::string ycoord;
148  getline(ss, xcoord, ' ');
149  getline(ss, ycoord, ' ');
150  KeyPoint kp;
151  kp.x = std::stoi(xcoord);
152  kp.y = std::stoi(ycoord);
153  kp.tracking_status = 1;
154  v.push_back(kp);
155  }
156  const int num_points = v.size();
157  array.resize(num_points);
158  for(int k = 0; k < num_points; ++k)
159  {
160  auto &keypoint = array.at(k);
161  keypoint = v[k];
162  }
163  return true;
164  }
165  else
166  {
167  std::cout << "Cannot open keypoints file " << fn << std::endl;
168  return false;
169  }
170  }
171 
172  /** Creates and Image and fills it with the ppm data from the file
173  *
174  * @param[in] fn PPM filename to be loaded
175  * @param[out] img Reference to an unintialised image instance
176  */
177  bool load_ppm(const std::string &fn, Image &img)
178  {
179  assert(!fn.empty());
180  PPMLoader ppm;
181  ppm.open(fn);
182  ppm.init_image(img, Format::U8);
183  img.allocator()->allocate();
184  if(ppm.is_open())
185  {
186  std::cout << "Reading image " << fn << std::endl;
187  ppm.fill_image(img);
188  return true;
189  }
190  else
191  {
192  std::cout << "Cannot open " << fn << std::endl;
193  return false;
194  }
195  }
196  /** Creates and Image and draws a square in the specified coordinares.
197  *
198  * @param[out] img Reference to an unintialised image instance
199  * @param[in] img_width Width of the image to be created
200  * @param[in] img_height Height of the image to be created
201  * @param[in] square_center_x Coordinate along x-axis to be used as the center for the square
202  * @param[in] square_center_y Coordinate along y-axis to be used as the center for the square
203  * @param[in] square_size Size in pixels to be used for the square
204  */
205  void init_img(Image &img, unsigned int img_width, unsigned int img_height,
206  unsigned int square_center_x, unsigned int square_center_y,
207  unsigned int square_size)
208  {
209  img.allocator()->init(TensorInfo(img_width, img_height, Format::U8));
210  img.allocator()->allocate();
211  const unsigned int square_half = square_size / 2;
212  // assert the square is in the bounds of the image
213  assert(square_center_x > square_half && square_center_x + square_half < img_width);
214  assert(square_center_y > square_half && square_center_y + square_half < img_height);
215  // get ptr to the top left pixel for the squeare
216  std::fill(img.buffer(), img.buffer() + img_width * img_height, 0);
217  for(unsigned int i = 0; i < square_size; ++i)
218  {
219  for(unsigned int j = 0; j < square_size; ++j)
220  {
221  uint8_t *ptr = img.ptr_to_element(Coordinates(square_center_x - square_half + j, square_center_y - square_half + i));
222  *ptr = 0xFF;
223  }
224  }
225  }
226  /** Prints an array of keypoints and an optional label
227  *
228  * @param[in] a Keypoint array to be printed
229  * @param[in] str Label to be printed before the array
230  */
231  void print_points(const KeyPointArray &a, const std::string &str = "")
232  {
233  std::cout << str << std::endl;
234  for(unsigned int k = 0; k < a.num_values(); ++k)
235  {
236  auto kp = a.at(k);
237  std::cout << "\t "
238  << " (x,y) = (" << kp.x << "," << kp.y << ")";
239  std::cout << " strength = " << kp.strength << " "
240  << " scale = " << kp.scale << " orientation " << kp.orientation << " status " << kp.tracking_status << " err = " << kp.error << std::endl;
241  }
242  }
243 
244  Pyramid pyr_1st{};
245  Pyramid pyr_2nd{};
246  NEGaussianPyramidHalf pyrf_1st{};
247  NEGaussianPyramidHalf pyrf_2nd{};
248  NEOpticalFlow optkf{};
249  Image src_1st{}, src_2nd{};
250  KeyPointArray input_points;
251  KeyPointArray output_points;
252  KeyPointArray point_estimates;
253 };
254 
255 /** Main program for optical flow test
256  *
257  * @param[in] argc Number of arguments
258  * @param[in] argv Arguments ( [optional] Path to PPM image to process )
259  */
260 int main(int argc, char **argv)
261 {
262  return utils::run_example<NeonOpticalFlowExample>(argc, argv);
263 }
constexpr float SCALE_PYRAMID_HALF
Constant value used to indicate a half-scale pyramid.
Definition: Types.h:112
int main(int argc, char **argv)
Main program for optical flow test.
float scale
Scale initialized to 0 by the corner detector.
Definition: Types.h:444
int32_t x
X coordinates.
Definition: Types.h:441
float orientation
Orientation initialized to 0 by the corner detector.
Definition: Types.h:445
void open(const std::string &filename) override
Open an image file and reads its metadata (Width, height)
Definition: ImageLoader.h:381
1 channel, 1 U8 per channel
int32_t tracking_status
Status initialized to 1 by the corner detector, set to 0 when the point is lost.
Definition: Types.h:446
void resize(size_t num)
Resizes the array to contain "num" elements.
Definition: IArray.h:128
std::stringstream ss(mlgo_str)
Terminate on whichever of the other conditions occurs first.
Includes all the Neon functions at once.
Copyright (c) 2017-2021 Arm Limited.
Basic implementation of the pyramid interface.
Definition: Pyramid.h:40
Keypoint type.
Definition: Types.h:439
Basic implementation of the IArray interface which allocates a static number of T values...
Definition: Array.h:36
float strength
Strength of the point.
Definition: Types.h:443
float error
Tracking error initialized to 0 by the corner detector.
Definition: Types.h:447
size_t num_values() const
Number of values currently stored in the array.
Definition: IArray.h:68
library fill(src, distribution, 0)
Basic function to execute gaussian pyramid with HALF scale factor.
Coordinates of an item.
Definition: Coordinates.h:37
Abstract Example class.
Definition: Utils.h:78
int stoi(const std::string &str, std::size_t *pos=0, NumericBase base=NumericBase::BASE_10)
Convert string values to integer.
Definition: StringSupport.h:55
Basic function to execute optical flow.
Definition: NEOpticalFlow.h:57
Structure to hold Image information.
Definition: helpers.h:659
int32_t y
Y coordinates.
Definition: Types.h:442
Store the Pyramid&#39;s metadata.
Definition: PyramidInfo.h:35
Borders are left undefined.
Store the tensor&#39;s metadata.
Definition: TensorInfo.h:45
PPM Image loader concrete implementation.
Definition: ImageLoader.h:367
virtual T & at(size_t index) const
Reference to the element of the array located at the given index.
Definition: IArray.h:117