12 #include <mrpt/3rdparty/do_opencv_includes.h> 28 template <
typename FEATLIST>
30 FEATLIST& featureList,
const CImage& cur_gray,
31 const float minimum_KLT_response,
const unsigned int KLT_response_half_win,
32 const unsigned int max_x,
const unsigned int max_y);
37 const float minimum_KLT_response,
const unsigned int KLT_response_half_win,
38 const unsigned int max_x,
const unsigned int max_y)
40 for (
auto& ft : featureList)
45 const unsigned int x = ft.keypoint.pt.x;
46 const unsigned int y = ft.keypoint.pt.y;
47 if (x > KLT_response_half_win && y > KLT_response_half_win &&
48 x < max_x && y < max_y)
50 ft.response = cur_gray.KLT_response(x, y, KLT_response_half_win);
54 if (ft.response < minimum_KLT_response)
67 template <
class FEAT_LIST>
69 FEAT_LIST& featureList,
const CImage& cur_gray,
70 const float minimum_KLT_response,
const unsigned int KLT_response_half_win,
71 const unsigned int max_x_,
const unsigned int max_y_)
73 if (featureList.empty())
return;
75 using pixel_coord_t =
typename FEAT_LIST::feature_t::pixel_coord_t;
76 const auto half_win =
static_cast<pixel_coord_t
>(KLT_response_half_win);
77 const auto max_x =
static_cast<pixel_coord_t
>(max_x_);
78 const auto max_y =
static_cast<pixel_coord_t
>(max_y_);
80 for (
int N = featureList.size() - 1; N >= 0; --N)
82 typename FEAT_LIST::feature_t& ft = featureList[N];
86 if (ft.pt.x > half_win && ft.pt.y > half_win && ft.pt.x < max_x &&
90 cur_gray.
KLT_response(ft.pt.x, ft.pt.y, KLT_response_half_win);
94 if (ft.response < minimum_KLT_response)
110 const float minimum_KLT_response,
const unsigned int KLT_response_half_win,
111 const unsigned int max_x,
const unsigned int max_y)
113 trackFeatures_checkResponses_impl_simple<TKeyPointList>(
114 featureList, cur_gray, minimum_KLT_response, KLT_response_half_win,
120 const float minimum_KLT_response,
const unsigned int KLT_response_half_win,
121 const unsigned int max_x,
const unsigned int max_y)
123 trackFeatures_checkResponses_impl_simple<TKeyPointfList>(
124 featureList, cur_gray, minimum_KLT_response, KLT_response_half_win,
128 template <
typename FEATLIST>
130 FEATLIST& featureList,
const CImage& cur_gray);
136 for (
auto& ft : featureList)
141 const size_t patch_width = ft.patch->getWidth();
142 const size_t patch_height = ft.patch->getHeight();
143 if (patch_width > 0 && patch_height > 0)
147 const int offset = (int)patch_width / 2;
149 cur_gray.extract_patch(
150 *ft.patch,
round(ft.keypoint.pt.x) - offset,
151 round(ft.keypoint.pt.y) - offset, patch_width,
154 catch (std::exception&)
164 [[maybe_unused]]
const CImage& cur_gray)
171 [[maybe_unused]]
const CImage& cur_gray)
176 template <
typename FEATLIST>
179 const std::vector<size_t>& sorted_indices,
const size_t nNewToCheck,
180 const size_t maxNumFeatures,
const float minimum_KLT_response_to_add,
181 const double threshold_sqr_dist_to_add_new,
const size_t patchSize,
187 const std::vector<size_t>& sorted_indices,
const size_t nNewToCheck,
188 const size_t maxNumFeatures,
const float minimum_KLT_response_to_add,
189 const double threshold_sqr_dist_to_add_new,
const size_t patchSize,
192 const TImageSize imgSize = cur_gray.getSize();
193 const int offset = (int)patchSize / 2 + 1;
194 const int w_off = int(imgSize.
x - offset);
195 const int h_off = int(imgSize.
y - offset);
197 for (
size_t i = 0; i < nNewToCheck && featureList.size() < maxNumFeatures;
200 const TKeyPoint& feat = new_feats[sorted_indices[i]];
202 if (feat.
response < minimum_KLT_response_to_add)
continue;
204 double min_dist_sqr =
square(10000);
206 if (!featureList.empty())
209 featureList.kdTreeClosestPoint2DsqrError(feat.
pt.x, feat.
pt.y);
212 if (min_dist_sqr > threshold_sqr_dist_to_add_new &&
213 feat.
pt.x > offset && feat.
pt.y > offset && feat.
pt.x < w_off &&
231 cur_gray.extract_patch(
233 round(feat.
pt.y) - offset, patchSize, patchSize);
236 featureList.emplace_back(std::move(ft));
241 template <
class FEAT_LIST>
244 const std::vector<size_t>& sorted_indices,
const size_t nNewToCheck,
245 const size_t maxNumFeatures,
const float minimum_KLT_response_to_add,
246 const double threshold_sqr_dist_to_add_new,
247 [[maybe_unused]]
const size_t patchSize,
252 featureList.getVector());
254 for (
size_t i = 0; i < nNewToCheck && featureList.size() < maxNumFeatures;
257 const TKeyPoint& feat = new_feats[sorted_indices[i]];
258 if (feat.
response < minimum_KLT_response_to_add)
break;
261 double min_dist_sqr = std::numeric_limits<double>::max();
263 if (!featureList.empty())
266 kdtree.kdTreeClosestPoint2DsqrError(feat.
pt.x, feat.
pt.y);
269 if (min_dist_sqr > threshold_sqr_dist_to_add_new)
272 featureList.emplace_back(feat.
pt.x, feat.
pt.y);
273 kdtree.mark_as_outdated();
276 typename FEAT_LIST::feature_t& newFeat = featureList.back();
278 newFeat.ID = ++max_feat_ID_at_input;
291 const std::vector<size_t>& sorted_indices,
const size_t nNewToCheck,
292 const size_t maxNumFeatures,
const float minimum_KLT_response_to_add,
293 const double threshold_sqr_dist_to_add_new,
const size_t patchSize,
296 trackFeatures_addNewFeats_simple_list<TKeyPointList>(
297 featureList, new_feats, sorted_indices, nNewToCheck, maxNumFeatures,
298 minimum_KLT_response_to_add, threshold_sqr_dist_to_add_new, patchSize,
299 cur_gray, max_feat_ID_at_input);
304 const std::vector<size_t>& sorted_indices,
const size_t nNewToCheck,
305 const size_t maxNumFeatures,
const float minimum_KLT_response_to_add,
306 const double threshold_sqr_dist_to_add_new,
const size_t patchSize,
309 trackFeatures_addNewFeats_simple_list<TKeyPointfList>(
310 featureList, new_feats, sorted_indices, nNewToCheck, maxNumFeatures,
311 minimum_KLT_response_to_add, threshold_sqr_dist_to_add_new, patchSize,
312 cur_gray, max_feat_ID_at_input);
316 template <
typename FEATLIST>
318 FEATLIST& trackedFeats,
const size_t img_width,
const size_t img_height,
319 const int MIN_DIST_MARGIN_TO_STOP_TRACKING);
321 template <
typename FEATLIST>
323 FEATLIST& trackedFeats,
const size_t img_width,
const size_t img_height,
324 const int MIN_DIST_MARGIN_TO_STOP_TRACKING)
326 if (trackedFeats.empty())
return 0;
328 std::vector<size_t> survival_idxs;
329 const size_t N = trackedFeats.size();
332 survival_idxs.reserve(N);
333 for (
size_t i = 0; i < N; i++)
335 const typename FEATLIST::feature_t& ft = trackedFeats[i];
341 const int x = ft.pt.x;
342 const int y = ft.pt.y;
343 if (x < MIN_DIST_MARGIN_TO_STOP_TRACKING ||
344 y < MIN_DIST_MARGIN_TO_STOP_TRACKING ||
345 x > static_cast<int>(
346 img_width - MIN_DIST_MARGIN_TO_STOP_TRACKING) ||
347 y >
static_cast<int>(
348 img_height - MIN_DIST_MARGIN_TO_STOP_TRACKING))
353 if (!eras) survival_idxs.push_back(i);
357 const size_t N2 = survival_idxs.size();
358 const size_t n_removed = N - N2;
359 for (
size_t i = 0; i < N2; i++)
361 if (survival_idxs[i] != i)
362 trackedFeats[i] = trackedFeats[survival_idxs[i]];
364 trackedFeats.resize(N2);
371 const size_t img_height,
const int MIN_DIST_MARGIN_TO_STOP_TRACKING)
373 return trackFeatures_deleteOOB_impl_simple_feat<TKeyPointList>(
374 trackedFeats, img_width, img_height, MIN_DIST_MARGIN_TO_STOP_TRACKING);
379 const size_t img_height,
const int MIN_DIST_MARGIN_TO_STOP_TRACKING)
381 return trackFeatures_deleteOOB_impl_simple_feat<TKeyPointfList>(
382 trackedFeats, img_width, img_height, MIN_DIST_MARGIN_TO_STOP_TRACKING);
387 CFeatureList& trackedFeats,
const size_t img_width,
const size_t img_height,
388 const int MIN_DIST_MARGIN_TO_STOP_TRACKING)
390 auto itFeat = trackedFeats.
begin();
391 size_t n_removed = 0;
392 while (itFeat != trackedFeats.
end())
399 const float x = itFeat->keypoint.pt.x;
400 const float y = itFeat->keypoint.pt.y;
401 if (x < MIN_DIST_MARGIN_TO_STOP_TRACKING ||
402 y < MIN_DIST_MARGIN_TO_STOP_TRACKING ||
403 x > (img_width - MIN_DIST_MARGIN_TO_STOP_TRACKING) ||
404 y > (img_height - MIN_DIST_MARGIN_TO_STOP_TRACKING))
411 itFeat = trackedFeats.
erase(itFeat);
424 [[maybe_unused]]
const CImage& old_img,
425 [[maybe_unused]]
const CImage& new_img,
443 template <
typename FEATLIST>
445 const CImage& old_img,
const CImage& new_img, FEATLIST& featureList)
449 const size_t img_width = new_img.
getWidth();
450 const size_t img_height = new_img.
getHeight();
455 if (!featureList.empty()) max_feat_ID_at_input = featureList.getMaxID();
459 m_timlog.enter(
"CGenericFeatureTracker.to_grayscale");
464 m_timlog.leave(
"CGenericFeatureTracker.to_grayscale");
469 m_newly_detected_feats.clear();
471 m_timlog.enter(
"CGenericFeatureTracker.trackFeatures_impl");
473 trackFeatures_impl(prev_gray, cur_gray, featureList);
475 m_timlog.leave(
"CGenericFeatureTracker.trackFeatures_impl");
480 const int check_KLT_response_every =
481 extra_params.getWithDefaultVal(
"check_KLT_response_every", 1);
482 const float minimum_KLT_response =
483 extra_params.getWithDefaultVal(
"minimum_KLT_response", 30.f);
484 const unsigned int KLT_response_half_win =
485 extra_params.getWithDefaultVal(
"KLT_response_half_win", 8U);
487 if (check_KLT_response_every > 0 &&
488 ++m_check_KLT_counter >=
size_t(check_KLT_response_every))
490 m_timlog.enter(
"CGenericFeatureTracker.check_KLT_responses");
491 m_check_KLT_counter = 0;
493 const unsigned int max_x = img_width - KLT_response_half_win;
494 const unsigned int max_y = img_height - KLT_response_half_win;
497 featureList, cur_gray, minimum_KLT_response, KLT_response_half_win,
500 m_timlog.leave(
"CGenericFeatureTracker.check_KLT_responses");
508 const bool remove_lost_features =
509 extra_params.getWithDefaultVal(
"remove_lost_features", 0) != 0;
511 if (remove_lost_features)
513 m_timlog.enter(
"CGenericFeatureTracker.OOB_remove");
515 static const int MIN_DIST_MARGIN_TO_STOP_TRACKING = 10;
518 featureList, img_width, img_height,
519 MIN_DIST_MARGIN_TO_STOP_TRACKING);
521 m_timlog.leave(
"CGenericFeatureTracker.OOB_remove");
523 last_execution_extra_info.num_deleted_feats = nRemoved;
527 last_execution_extra_info.num_deleted_feats = 0;
533 const int update_patches_every =
534 extra_params.getWithDefaultVal(
"update_patches_every", 0);
536 if (update_patches_every > 0 &&
537 ++m_update_patches_counter >=
size_t(update_patches_every))
540 m_timlog,
"CGenericFeatureTracker.update_patches");
542 m_update_patches_counter = 0;
552 const bool add_new_features =
553 extra_params.getWithDefaultVal(
"add_new_features", 1) != 0;
554 const double threshold_dist_to_add_new =
555 extra_params.getWithDefaultVal(
"add_new_feat_min_separation", 15);
559 if (add_new_features)
562 m_timlog,
"CGenericFeatureTracker.add_new_features");
566 if (m_newly_detected_feats.empty())
577 m_newly_detected_feats.clear();
578 m_newly_detected_feats.reserve(new_feats.
size());
582 m_newly_detected_feats.push_back(
TKeyPoint(f.keypoint));
587 const size_t N = m_newly_detected_feats.size();
588 last_execution_extra_info.raw_FAST_feats_detected = N;
591 const size_t desired_num_features = extra_params.getWithDefaultVal(
592 "desired_num_features_adapt",
593 size_t((img_width * img_height) >> 9));
594 updateAdaptiveNewFeatsThreshold(N, desired_num_features);
598 const unsigned int max_x = img_width - KLT_response_half_win;
599 const unsigned int max_y = img_height - KLT_response_half_win;
600 for (
size_t i = 0; i < N; i++)
602 const unsigned int x = m_newly_detected_feats[i].pt.x;
603 const unsigned int y = m_newly_detected_feats[i].pt.y;
604 if (x > KLT_response_half_win && y > KLT_response_half_win &&
605 x < max_x && y < max_y)
606 m_newly_detected_feats[i].response =
609 m_newly_detected_feats[i].response = 0;
616 std::vector<size_t> sorted_indices(N);
617 for (
size_t i = 0; i < N; i++) sorted_indices[i] = i;
620 sorted_indices.begin(), sorted_indices.end(),
627 const size_t nNewToCheck = std::min(
size_t(1500), N);
628 const double threshold_sqr_dist_to_add_new =
629 square(threshold_dist_to_add_new);
630 const size_t maxNumFeatures =
631 extra_params.getWithDefaultVal(
"add_new_feat_max_features", 100U);
632 const size_t patchSize =
633 extra_params.getWithDefaultVal(
"add_new_feat_patch_size", 0U);
634 const float minimum_KLT_response_to_add =
635 extra_params.getWithDefaultVal(
"minimum_KLT_response_to_add", 70.f);
639 featureList, m_newly_detected_feats, sorted_indices, nNewToCheck,
640 maxNumFeatures, minimum_KLT_response_to_add,
641 threshold_sqr_dist_to_add_new, patchSize, cur_gray,
642 max_feat_ID_at_input);
650 internal_trackFeatures<TKeyPointList>(old_img, new_img, featureList);
656 internal_trackFeatures<TKeyPointfList>(old_img, new_img, featureList);
660 const size_t nNewlyDetectedFeats,
const size_t desired_num_features)
662 const size_t hysteresis_min_num_feats = desired_num_features * 0.9;
663 const size_t hysteresis_max_num_feats = desired_num_features * 1.1;
665 if (nNewlyDetectedFeats < hysteresis_min_num_feats)
666 m_detector_adaptive_thres = std::max(
668 m_detector_adaptive_thres - 1.0,
669 m_detector_adaptive_thres * 0.8));
670 else if (nNewlyDetectedFeats > hysteresis_max_num_feats)
671 m_detector_adaptive_thres = std::max(
672 m_detector_adaptive_thres + 1.0, m_detector_adaptive_thres * 1.2);
void trackFeatures_updatePatch< TKeyPointList >([[maybe_unused]] TKeyPointList &featureList, [[maybe_unused]] const CImage &cur_gray)
void trackFeatures_addNewFeats< CFeatureList >(CFeatureList &featureList, const TKeyPointList &new_feats, const std::vector< size_t > &sorted_indices, const size_t nNewToCheck, const size_t maxNumFeatures, const float minimum_KLT_response_to_add, const double threshold_sqr_dist_to_add_new, const size_t patchSize, const CImage &cur_gray, TFeatureID &max_feat_ID_at_input)
Helper class: KD-tree search class for vector<KeyPoint>: Call mark_as_outdated() to force rebuilding ...
uint64_t TFeatureID
Definition of a feature ID.
void trackFeatures(const mrpt::img::CImage &old_img, const mrpt::img::CImage &new_img, TKeyPointList &inout_featureList)
Perform feature tracking from "old_img" to "new_img", with a (possibly empty) list of previously trac...
#define THROW_EXCEPTION(msg)
A safe way to call enter() and leave() of a mrpt::system::CTimeLogger upon construction and destructi...
size_t trackFeatures_deleteOOB_impl_simple_feat(FEATLIST &trackedFeats, const size_t img_width, const size_t img_height, const int MIN_DIST_MARGIN_TO_STOP_TRACKING)
TKeyPointMethod type
Keypoint method used to detect this feature.
TKeyPoint_templ< mrpt::img::TPixelCoord > TKeyPoint
Simple structure for image key points.
void internal_trackFeatures(const mrpt::img::CImage &old_img, const mrpt::img::CImage &new_img, FEATLIST &inout_featureList)
Perform feature tracking from "old_img" to "new_img", with a (possibly empty) list of previously trac...
Unable to track this feature (mismatch is too high for the given tracking window: lack of texture...
size_t getHeight() const override
Returns the height of the image in pixels.
This file implements miscelaneous matrix and matrix/vector operations, and internal functions in mrpt...
void trackFeatures_checkResponses< TKeyPointfList >(TKeyPointfList &featureList, const CImage &cur_gray, const float minimum_KLT_response, const unsigned int KLT_response_half_win, const unsigned int max_x, const unsigned int max_y)
TFeatureID ID
ID of the feature.
void trackFeatures_addNewFeats_simple_list(FEAT_LIST &featureList, const TKeyPointList &new_feats, const std::vector< size_t > &sorted_indices, const size_t nNewToCheck, const size_t maxNumFeatures, const float minimum_KLT_response_to_add, const double threshold_sqr_dist_to_add_new, [[maybe_unused]] const size_t patchSize, [[maybe_unused]] const CImage &cur_gray, TFeatureID &max_feat_ID_at_input)
Simple structure for image key points.
void trackFeatures_checkResponses_impl_simple(FEAT_LIST &featureList, const CImage &cur_gray, const float minimum_KLT_response, const unsigned int KLT_response_half_win, const unsigned int max_x_, const unsigned int max_y_)
A helper struct to sort keypoints by their response: It can be used with these types: ...
virtual void trackFeatures_impl(const mrpt::img::CImage &old_img, const mrpt::img::CImage &new_img, TKeyPointfList &inout_featureList)
The tracking method implementation, to be implemented in children classes.
Inactive (right after detection, and before being tried to track)
This base provides a set of functions for maths stuff.
size_t getWidth() const override
Returns the width of the image in pixels.
float KLT_response(const unsigned int x, const unsigned int y, const unsigned int half_window_size) const
Compute the KLT response at a given pixel (x,y) - Only for grayscale images (for efficiency it avoids...
std::optional< mrpt::img::CImage > patch
A patch of the image surrounding the feature.
Feature fell Out Of Bounds (out of the image limits, too close to image borders)
void updateAdaptiveNewFeatsThreshold(const size_t nNewlyDetectedFeats, const size_t desired_num_features)
Adapts the threshold m_detector_adaptive_thres according to the real and desired number of features j...
A pair (x,y) of pixel coordinates (integer resolution).
Classes for computer vision, detectors, features, etc.
A generic 2D feature from an image, extracted with CFeatureExtraction Each feature may have one or mo...
iterator erase(const iterator &it)
FAST feature detector, OpenCV's implementation ("Faster and better: A machine learning approach to...
A list of visual features, to be used as output by detectors, as input/output by trackers, etc.
void trackFeatures_addNewFeats< TKeyPointList >(TKeyPointList &featureList, const TKeyPointList &new_feats, const std::vector< size_t > &sorted_indices, const size_t nNewToCheck, const size_t maxNumFeatures, const float minimum_KLT_response_to_add, const double threshold_sqr_dist_to_add_new, const size_t patchSize, const CImage &cur_gray, TFeatureID &max_feat_ID_at_input)
return_t square(const num_t x)
Inline function for the square of a number.
float response
A measure of the "goodness" of the feature.
This is the global namespace for all Mobile Robot Programming Toolkit (MRPT) libraries.
void trackFeatures_checkResponses(FEATLIST &featureList, const CImage &cur_gray, const float minimum_KLT_response, const unsigned int KLT_response_half_win, const unsigned int max_x, const unsigned int max_y)
size_t trackFeatures_deleteOOB(FEATLIST &trackedFeats, const size_t img_width, const size_t img_height, const int MIN_DIST_MARGIN_TO_STOP_TRACKING)
uint8_t octave
The image octave the image was found in: 0=original image, 1=1/2 image, 2=1/4 image, etc.
void trackFeatures_updatePatch< CFeatureList >(CFeatureList &featureList, const CImage &cur_gray)
void trackFeatures_addNewFeats(FEATLIST &featureList, const TKeyPointList &new_feats, const std::vector< size_t > &sorted_indices, const size_t nNewToCheck, const size_t maxNumFeatures, const float minimum_KLT_response_to_add, const double threshold_sqr_dist_to_add_new, const size_t patchSize, const CImage &cur_gray, TFeatureID &max_feat_ID_at_input)
Feature correctly tracked.
void trackFeatures_checkResponses< CFeatureList >(CFeatureList &featureList, const CImage &cur_gray, const float minimum_KLT_response, const unsigned int KLT_response_half_win, const unsigned int max_x, const unsigned int max_y)
float orientation
Main orientation of the feature.
Functions for estimating the optimal transformation between two frames of references given measuremen...
float response
A measure of the "goodness" of the feature (typically, the KLT_response value)
void trackFeatures_updatePatch< TKeyPointfList >([[maybe_unused]] TKeyPointfList &featureList, [[maybe_unused]] const CImage &cur_gray)
uint16_t patchSize
Size of the patch (patchSize x patchSize) (it must be an odd number)
void trackFeatures_updatePatch(FEATLIST &featureList, const CImage &cur_gray)
size_t trackFeatures_deleteOOB(CFeatureList &trackedFeats, const size_t img_width, const size_t img_height, const int MIN_DIST_MARGIN_TO_STOP_TRACKING)
void trackFeatures_addNewFeats< TKeyPointfList >(TKeyPointfList &featureList, const TKeyPointList &new_feats, const std::vector< size_t > &sorted_indices, const size_t nNewToCheck, const size_t maxNumFeatures, const float minimum_KLT_response_to_add, const double threshold_sqr_dist_to_add_new, const size_t patchSize, const CImage &cur_gray, TFeatureID &max_feat_ID_at_input)
pixel_coords_t pt
Coordinates in the image.
void trackFeatures_checkResponses< TKeyPointList >(TKeyPointList &featureList, const CImage &cur_gray, const float minimum_KLT_response, const unsigned int KLT_response_half_win, const unsigned int max_x, const unsigned int max_y)
A class for storing images as grayscale or RGB bitmaps.
int round(const T value)
Returns the closer integer (int) to x.