MRPT  1.9.9
CFeatureExtraction_ORB.cpp
Go to the documentation of this file.
1 /* +------------------------------------------------------------------------+
2  | Mobile Robot Programming Toolkit (MRPT) |
3  | https://www.mrpt.org/ |
4  | |
5  | Copyright (c) 2005-2019, Individual contributors, see AUTHORS file |
6  | See: https://www.mrpt.org/Authors - All rights reserved. |
7  | Released under BSD License. See: https://www.mrpt.org/License |
8  +------------------------------------------------------------------------+ */
9 
10 #include "vision-precomp.h" // Precompiled headers
11 
13 
14 // Universal include for all versions of OpenCV
15 #include <mrpt/3rdparty/do_opencv_includes.h>
16 
17 using namespace mrpt;
18 using namespace mrpt::vision;
19 using namespace mrpt::img;
20 using namespace mrpt::system;
21 using namespace mrpt::img;
22 using namespace std;
23 
25  const mrpt::img::CImage& inImg, CFeatureList& feats,
26  const unsigned int init_ID, const unsigned int nDesiredFeatures,
27  const TImageROI& ROI)
28 {
29  MRPT_UNUSED_PARAM(ROI);
31 
32  mrpt::system::CTimeLoggerEntry tle(profiler, "extractFeaturesORB");
33 
34 #if MRPT_HAS_OPENCV
35  using namespace cv;
36 
37  vector<KeyPoint> cv_feats; // OpenCV keypoint output vector
38  Mat cv_descs; // OpenCV descriptor output
39 
40  const bool use_precomputed_feats = feats.size() > 0;
41 
42  if (use_precomputed_feats)
43  {
44  cv_feats.resize(feats.size());
45  for (size_t k = 0; k < cv_feats.size(); ++k)
46  {
47  cv_feats[k].pt.x = feats[k].keypoint.pt.x;
48  cv_feats[k].pt.y = feats[k].keypoint.pt.y;
49  }
50  }
51 
52  // Make sure we operate on a gray-scale version of the image:
53  const CImage inImg_gray(inImg, FAST_REF_OR_CONVERT_TO_GRAY);
54  const Mat& cvImg = inImg_gray.asCvMatRef();
55 
56  // The detector and descriptor
57  profiler.enter("extractFeaturesORB.openCV_detectAndCompute");
58 
59 #if MRPT_OPENCV_VERSION_NUM < 0x300
60  Ptr<Feature2D> orb = Algorithm::create<Feature2D>("Feature2D.ORB");
61  orb->operator()(cvImg, Mat(), cv_feats, cv_descs, use_precomputed_feats);
62 #else
63  const size_t n_feats_2_extract =
64  nDesiredFeatures == 0 ? 1000 : 3 * nDesiredFeatures;
65  Ptr<cv::ORB> orb = cv::ORB::create(
66  n_feats_2_extract, options.ORBOptions.scale_factor,
67  options.ORBOptions.n_levels);
68  orb->detectAndCompute(
69  cvImg, Mat(), cv_feats, cv_descs, use_precomputed_feats);
70 #endif
71  profiler.leave("extractFeaturesORB.openCV_detectAndCompute");
72 
73  CTimeLoggerEntry tle2(profiler, "extractFeaturesORB.fillFeatsStruct");
74 
75  const size_t n_feats = cv_feats.size();
76 
77  // if we had input features, just convert cv_feats to CFeatures and return
78  const unsigned int patch_size_2 = options.patchSize / 2;
79  unsigned int f_id = init_ID;
80  if (use_precomputed_feats)
81  {
82  for (size_t k = 0; k < n_feats; ++k)
83  {
84  feats[k].descriptors.ORB->resize(cv_descs.cols);
85  for (int m = 0; m < cv_descs.cols; ++m)
86  (*feats[k].descriptors.ORB)[m] = cv_descs.at<uchar>(k, m);
87 
88  /*
89  feats[k].response = cv_feats[k].response;
90  feats[k].scale = cv_feats[k].size;
91  feats[k].angle = cv_feats[k].orientation;
92  feats[k].ID = f_id++;
93  */
94  feats[k].type = featORB;
95 
96  if (options.ORBOptions.extract_patch && options.patchSize > 0)
97  {
98  inImg.extract_patch(
99  *feats[k].patch,
100  round(feats[k].keypoint.pt.x) - patch_size_2,
101  round(feats[k].keypoint.pt.y) - patch_size_2,
102  options.patchSize, options.patchSize);
103  }
104  }
105  return;
106  }
107 
108  // 1) Sort the fearues by "response": It's ~100 times faster to sort a list
109  // of
110  // indices "sorted_indices" than sorting directly the actual list of
111  // features "cv_feats"
112  std::vector<size_t> sorted_indices(n_feats);
113  for (size_t i = 0; i < n_feats; i++) sorted_indices[i] = i;
114  std::sort(
115  sorted_indices.begin(), sorted_indices.end(),
117 
118  // 2) Filter by "min-distance" (in options.ORBOptions.min_distance)
119  // 3) Convert to MRPT CFeatureList format.
120  // Steps 2 & 3 are done together in the while() below.
121  // The "min-distance" filter is done by means of a 2D binary matrix where
122  // each cell is marked when one
123  // feature falls within it. This is not exactly the same than a pure
124  // "min-distance" but is pretty close
125  // and for large numbers of features is much faster than brute force search
126  // of kd-trees.
127  // (An intermediate approach would be the creation of a mask image updated
128  // for each accepted feature, etc.)
129 
130  const bool do_filter_min_dist = options.ORBOptions.min_distance > 1;
131  const unsigned int occupied_grid_cell_size =
132  options.ORBOptions.min_distance / 2.0;
133  const float occupied_grid_cell_size_inv = 1.0f / occupied_grid_cell_size;
134 
135  unsigned int grid_lx =
136  !do_filter_min_dist
137  ? 1
138  : (unsigned int)(1 + inImg.getWidth() * occupied_grid_cell_size_inv);
139  unsigned int grid_ly =
140  !do_filter_min_dist
141  ? 1
142  : (unsigned int)(1 + inImg.getHeight() * occupied_grid_cell_size_inv);
143 
144  mrpt::math::CMatrixBool occupied_sections(
145  grid_lx, grid_ly); // See the comments above for an explanation.
146  occupied_sections.fill(false);
147 
148  const size_t n_max_feats = nDesiredFeatures > 0
149  ? std::min(size_t(nDesiredFeatures), n_feats)
150  : n_feats;
151 
152  if (!options.addNewFeatures) feats.clear();
153  // feats.reserve( feats.size() + n_max_feats );
154 
155  const size_t imgH = inImg.getHeight();
156  const size_t imgW = inImg.getWidth();
157  size_t k = 0;
158  size_t c_feats = 0;
159  while (c_feats < n_max_feats && k < n_feats)
160  {
161  const size_t idx = sorted_indices[k++];
162  const KeyPoint& kp = cv_feats[idx];
163  if (options.ORBOptions.extract_patch && options.patchSize > 0)
164  {
165  // check image boundaries for extracting the patch
166  const int xBorderInf = (int)floor(kp.pt.x - patch_size_2);
167  const int xBorderSup = (int)floor(kp.pt.x + patch_size_2);
168  const int yBorderInf = (int)floor(kp.pt.y - patch_size_2);
169  const int yBorderSup = (int)floor(kp.pt.y + patch_size_2);
170 
171  if (!(xBorderSup < (int)imgW && xBorderInf > 0 &&
172  yBorderSup < (int)imgH && yBorderInf > 0))
173  continue; // nope, skip.
174  }
175 
176  if (do_filter_min_dist)
177  {
178  // Check the min-distance:
179  const auto sect_ix = size_t(kp.pt.x * occupied_grid_cell_size_inv);
180  const auto sect_iy = size_t(kp.pt.y * occupied_grid_cell_size_inv);
181 
182  if (occupied_sections(sect_ix, sect_iy))
183  continue; // Already occupied! skip.
184 
185  // Mark section as occupied
186  occupied_sections(sect_ix, sect_iy) = true;
187  if (sect_ix > 0) occupied_sections(sect_ix - 1, sect_iy) = true;
188  if (sect_iy > 0) occupied_sections(sect_ix, sect_iy - 1) = true;
189  if (sect_ix < grid_lx - 1)
190  occupied_sections(sect_ix + 1, sect_iy) = true;
191  if (sect_iy < grid_ly - 1)
192  occupied_sections(sect_ix, sect_iy + 1) = true;
193  }
194 
195  // All tests passed: add new feature:
196  CFeature ft;
197  ft.type = featORB;
198  ft.keypoint.ID = f_id++;
199  ft.keypoint.pt.x = kp.pt.x;
200  ft.keypoint.pt.y = kp.pt.y;
201  ft.response = kp.response;
202  ft.orientation = kp.angle;
203  ft.keypoint.octave = kp.octave;
204  ft.patchSize = 0;
205 
206  // descriptor
207  ft.descriptors.ORB.emplace();
208  ft.descriptors.ORB->resize(cv_descs.cols);
209  for (int m = 0; m < cv_descs.cols; ++m)
210  (*ft.descriptors.ORB)[m] = cv_descs.at<uchar>(idx, m);
211 
212  if (options.ORBOptions.extract_patch && options.patchSize > 0)
213  {
214  ft.patchSize = options.patchSize; // The size of the feature patch
215 
216  ft.patch.emplace();
217  inImg.extract_patch(
218  *ft.patch, round(kp.pt.x) - patch_size_2,
219  round(kp.pt.y) - patch_size_2, options.patchSize,
220  options.patchSize); // Image patch surronding the feature
221  }
222 
223  feats.emplace_back(std::move(ft));
224  c_feats++;
225  }
226 #endif
227  MRPT_END
228 }
229 
231  const CImage& in_img, CFeatureList& in_features)
232 {
233 #if MRPT_HAS_OPENCV
234  using namespace cv;
236  profiler, "internal_computeORBDescriptors");
237 
238  const size_t n_feats = in_features.size();
239  const CImage inImg_gray(in_img, FAST_REF_OR_CONVERT_TO_GRAY);
240 
241  // convert from CFeatureList to vector<KeyPoint>
242  vector<KeyPoint> cv_feats(n_feats);
243  for (size_t k = 0; k < n_feats; ++k)
244  {
245  KeyPoint& kp = cv_feats[k];
246  kp.pt.x = in_features[k].keypoint.pt.x;
247  kp.pt.y = in_features[k].keypoint.pt.y;
248  kp.angle = in_features[k].orientation;
249  kp.size = in_features[k].keypoint.octave;
250  } // end-for
251 
252  const Mat& cvImg = inImg_gray.asCvMatRef();
253  Mat cv_descs;
254 
255 #if MRPT_OPENCV_VERSION_NUM < 0x300
256  Ptr<Feature2D> orb = Algorithm::create<Feature2D>("Feature2D.ORB");
257  orb->operator()(
258  cvImg, cv::noArray(), cv_feats, cv_descs,
259  true /* use_precomputed_feats */);
260 #else
261  Ptr<cv::ORB> orb = cv::ORB::create(
262  n_feats, options.ORBOptions.scale_factor, options.ORBOptions.n_levels);
263  orb->detectAndCompute(
264  cvImg, cv::noArray(), cv_feats, cv_descs,
265  true /* use_precomputed_feats */);
266 #endif
267 
268  // add descriptor to CFeatureList
269  for (size_t k = 0; k < n_feats; ++k)
270  {
271  in_features[k].descriptors.ORB.emplace();
272  auto& orb_desc = *in_features[k].descriptors.ORB;
273  orb_desc.resize(cv_descs.cols);
274  for (int i = 0; i < cv_descs.cols; ++i)
275  orb_desc[i] = cv_descs.at<uchar>(k, i);
276 
277  } // end-for
278 #endif
279 
280 } // end-internal_computeORBImageDescriptors
#define MRPT_START
Definition: exceptions.h:241
std::optional< std::vector< uint8_t > > ORB
ORB feature descriptor.
Definition: CFeature.h:135
A safe way to call enter() and leave() of a mrpt::system::CTimeLogger upon construction and destructi...
TKeyPointMethod type
Keypoint method used to detect this feature.
Definition: CFeature.h:73
void fill(const Scalar &val)
size_t size() const
Definition: CFeature.h:352
cv::Mat & asCvMatRef()
Get a reference to the internal cv::Mat, which can be resized, etc.
Definition: CImage.cpp:234
size_t getHeight() const override
Returns the height of the image in pixels.
Definition: CImage.cpp:878
STL namespace.
TFeatureID ID
ID of the feature.
Definition: TKeyPoint.h:39
Definition: img/CImage.h:22
A helper struct to sort keypoints by their response: It can be used with these types: ...
Definition: TKeyPoint.h:288
A structure for defining a ROI within an image.
size_t getWidth() const override
Returns the width of the image in pixels.
Definition: CImage.cpp:847
void resize(size_t N)
Definition: CFeature.h:358
TKeyPointf keypoint
Definition: CFeature.h:64
std::optional< mrpt::img::CImage > patch
A patch of the image surrounding the feature.
Definition: CFeature.h:67
void internal_computeORBDescriptors(const mrpt::img::CImage &in_img, CFeatureList &in_features)
Compute the ORB descriptor of the provided features into the input image.
Classes for computer vision, detectors, features, etc.
Definition: CDifodo.h:17
A generic 2D feature from an image, extracted with CFeatureExtraction Each feature may have one or mo...
Definition: CFeature.h:53
void extractFeaturesORB(const mrpt::img::CImage &img, CFeatureList &feats, const unsigned int init_ID=0, const unsigned int nDesiredFeatures=0, const TImageROI &ROI=TImageROI())
Extract features from the image based on the ORB method.
A list of visual features, to be used as output by detectors, as input/output by trackers, etc.
Definition: CFeature.h:275
float response
A measure of the "goodness" of the feature.
Definition: CFeature.h:79
This is the global namespace for all Mobile Robot Programming Toolkit (MRPT) libraries.
TDescriptors descriptors
Definition: CFeature.h:157
#define MRPT_END
Definition: exceptions.h:245
uint8_t octave
The image octave the image was found in: 0=original image, 1=1/2 image, 2=1/4 image, etc.
Definition: TKeyPoint.h:49
void extract_patch(CImage &patch, const unsigned int col=0, const unsigned int row=0, const unsigned int width=1, const unsigned int height=1) const
Extract a patch from this image, saveing it into "patch" (its previous contents will be overwritten)...
Definition: CImage.cpp:1195
float orientation
Main orientation of the feature.
Definition: CFeature.h:81
This template class provides the basic functionality for a general 2D any-size, resizable container o...
uint16_t patchSize
Size of the patch (patchSize x patchSize) (it must be an odd number)
Definition: CFeature.h:70
void emplace_back(CFeature &&f)
Definition: CFeature.h:364
ORB detector and descriptor, OpenCV&#39;s implementation.
pixel_coords_t pt
Coordinates in the image.
Definition: TKeyPoint.h:36
A class for storing images as grayscale or RGB bitmaps.
Definition: img/CImage.h:147
#define MRPT_UNUSED_PARAM(a)
Determines whether this is an X86 or AMD64 platform.
Definition: common.h:186
int round(const T value)
Returns the closer integer (int) to x.
Definition: round.h:24



Page generated by Doxygen 1.8.14 for MRPT 1.9.9 Git: ce444d842 Fri Dec 6 19:35:10 2019 +0100 at vie dic 6 19:45:12 CET 2019