From 6dad7630d1a2d38200766dbad1e9140abcbabba9 Mon Sep 17 00:00:00 2001 From: kalwalt Date: Thu, 30 May 2024 15:44:42 +0200 Subject: [PATCH] new processFrame2 function under testing --- WebARKit/CMakeLists.txt | 4 + .../WebARKitOpticalTracking/TrackedPoint.cpp | 60 +++++ .../TrackingPointSelector.cpp | 194 +++++++++++++++ .../WebARKitConfig.cpp | 2 + .../WebARKitTracker.cpp | 223 ++++++++++++------ .../WebARKitOpticalTracking/TrackedPoint.h | 61 +++++ .../TrackingPointSelector.h | 94 ++++++++ .../WebARKitOpticalTracking/WebARKitConfig.h | 4 +- .../WebARKitOpticalTracking/WebARKitUtils.h | 43 ++-- 9 files changed, 602 insertions(+), 83 deletions(-) create mode 100644 WebARKit/WebARKitTrackers/WebARKitOpticalTracking/TrackedPoint.cpp create mode 100644 WebARKit/WebARKitTrackers/WebARKitOpticalTracking/TrackingPointSelector.cpp create mode 100644 WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/TrackedPoint.h create mode 100644 WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/TrackingPointSelector.h diff --git a/WebARKit/CMakeLists.txt b/WebARKit/CMakeLists.txt index 23d82ea..c4e67ea 100644 --- a/WebARKit/CMakeLists.txt +++ b/WebARKit/CMakeLists.txt @@ -29,6 +29,8 @@ FetchContent_MakeAvailable(build_opencv) get_filename_component(PARENT_DIR ./ ABSOLUTE) set(WEBARKIT_HEADERS +${PARENT_DIR}/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/TrackedPoint.h +${PARENT_DIR}/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/TrackingPointSelector.h ${PARENT_DIR}/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/WebARKitConfig.h ${PARENT_DIR}/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking//WebARKitEnums.h ${PARENT_DIR}/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/WebARKitTracker.h @@ -41,6 +43,8 @@ ${PARENT_DIR}/include/WebARKitPattern.h ) set(SOURCE +${PARENT_DIR}/WebARKitTrackers/WebARKitOpticalTracking/TrackedPoint.cpp +${PARENT_DIR}/WebARKitTrackers/WebARKitOpticalTracking/TrackingPointSelector.cpp ${PARENT_DIR}/WebARKitTrackers/WebARKitOpticalTracking/WebARKitConfig.cpp ${PARENT_DIR}/WebARKitTrackers/WebARKitOpticalTracking/WebARKitTracker.cpp ${PARENT_DIR}/WebARKitCamera.cpp diff --git a/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/TrackedPoint.cpp b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/TrackedPoint.cpp new file mode 100644 index 0000000..e2d6223 --- /dev/null +++ b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/TrackedPoint.cpp @@ -0,0 +1,60 @@ +/* + * TrackedPoint.cpp + * artoolkitX + * + * This file is part of artoolkitX. + * + * artoolkitX is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * artoolkitX is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with artoolkitX. If not, see . + * + * As a special exception, the copyright holders of this library give you + * permission to link this library with independent modules to produce an + * executable, regardless of the license terms of these independent modules, and to + * copy and distribute the resulting executable under terms of your choice, + * provided that you also meet, for each linked independent module, the terms and + * conditions of the license of that module. An independent module is a module + * which is neither derived from nor based on this library. If you modify this + * library, you may extend this exception to your version of the library, but you + * are not obligated to do so. If you do not wish to do so, delete this exception + * statement from your version. + * + * Copyright 2018 Realmax, Inc. + * Copyright 2015 Daqri, LLC. + * Copyright 2010-2015 ARToolworks, Inc. + * + * Author(s): Philip Lamb, Daniel Bell. + * Modified for WebARKit by @kalwalt - Walter Perdan - 2024 + * + */ + +#include + +bool TrackedPoint::IsTracking() +{ + return tracking; +} + +void TrackedPoint::SetTracking(bool newTracking) +{ + tracking = newTracking; +} + +bool TrackedPoint::IsSelected() +{ + return selected; +} + +void TrackedPoint::SetSelected(bool newSelected) +{ + selected = newSelected; +} diff --git a/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/TrackingPointSelector.cpp b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/TrackingPointSelector.cpp new file mode 100644 index 0000000..e77a5fc --- /dev/null +++ b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/TrackingPointSelector.cpp @@ -0,0 +1,194 @@ +/* + * TrackingPointSelector.cpp + * artoolkitX + * + * This file is part of artoolkitX. + * + * artoolkitX is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * artoolkitX is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with artoolkitX. If not, see . + * + * As a special exception, the copyright holders of this library give you + * permission to link this library with independent modules to produce an + * executable, regardless of the license terms of these independent modules, and to + * copy and distribute the resulting executable under terms of your choice, + * provided that you also meet, for each linked independent module, the terms and + * conditions of the license of that module. An independent module is a module + * which is neither derived from nor based on this library. If you modify this + * library, you may extend this exception to your version of the library, but you + * are not obligated to do so. If you do not wish to do so, delete this exception + * statement from your version. + * + * Copyright 2018 Realmax, Inc. + * Copyright 2015 Daqri, LLC. + * Copyright 2010-2015 ARToolworks, Inc. + * + * Author(s): Philip Lamb, Daniel Bell. + * Modified for WebARKit by @kalwalt - Walter Perdan - 2024 + * + */ + +#include + +TrackingPointSelector::TrackingPointSelector() +{ +} + +TrackingPointSelector::TrackingPointSelector(std::vector pts, int width, int height, int markerTemplateWidth) : + _reset(false), + _pts(pts) +{ + DistributeBins(width, height, markerTemplateWidth); +} + +/** + @brief Iterates over \_pts and for each one, provided it doesn't intersect the image border, + creates a TrackedPoint representing a template () with a serially-increasing id from 0, and puts + it into the trackingPointsBin structure (a vector of pairs of (binIndex, trackingPoint). + */ +void TrackingPointSelector::DistributeBins(int width, int height, int markerTemplateWidth) +{ + int numberOfBins = 10; + + // Split width and height dimensions into 10 bins each, for total of 100 bins. + int totalXBins = width/numberOfBins; + int totalYBins = height/numberOfBins; + // Init empty bins. + for (int i = 0; i < (numberOfBins * numberOfBins); i++) { + trackingPointBin.insert(std::pair >(i, std::vector())); + } + + // Iterate the points and add points to each bin. + for (int i = 0, id = 0; i < _pts.size(); i++) { + int bx = (int)_pts[i].x/totalXBins; + int by = (int)_pts[i].y/totalYBins; + int index = bx + (by * numberOfBins); + + cv::Rect templateRoi = cv::Rect(_pts[i].x - markerTemplateWidth, _pts[i].y - markerTemplateWidth, markerTemplateWidth*2, markerTemplateWidth*2); + bool is_inside = (templateRoi & cv::Rect(0, 0, width, height)) == templateRoi; // templateRoi must not intersect image boundary. + if (is_inside) { + TrackedPoint newPt; + newPt.id = id; + newPt.pt = _pts[i]; + newPt.pt3d = cv::Point3f(_pts[i].x, _pts[i].y, 0); + newPt.markerRoi = templateRoi; + trackingPointBin[index].push_back(newPt); + id++; + } + } +} + +void TrackingPointSelector::SetHomography(cv::Mat newHomography) +{ + _homography = newHomography; +} + +/// @return 3x3 cv::Mat (of type CV_64FC1, i.e. double) containing the homography. +cv::Mat TrackingPointSelector::GetHomography() +{ + return _homography; +} + +void TrackingPointSelector::UpdatePointStatus(std::vector status) +{ + int index = 0; + for (std::vector::iterator it = _selectedPts.begin(); it != _selectedPts.end(); ++it) { + if (it->tracking) { + it->SetTracking((int)status[index++]); + } + } +} + +void TrackingPointSelector::ResetSelection() +{ + _reset = true; +} + +std::vector TrackingPointSelector::GetInitialFeatures() +{ + if (!_reset) return GetTrackedFeatures(); + _reset = false; + + // Reset state of all points to not selected and not tracking. + _selectedPts.clear(); + for (auto &bin : trackingPointBin) { + for (auto &trackPt : bin.second) { + trackPt.SetSelected(false); + trackPt.SetTracking(false); + } + } + + // Selects a random template from each bin for tracking. + std::vector ret; + for (auto &bin : trackingPointBin) { + size_t pointCount = bin.second.size(); + if (pointCount > 0) { // If there are points in the bin. + // Select a random point from the bin. + int tIndex = pointCount > 1 ? rng.uniform(0, static_cast(bin.second.size())) : 0; + bin.second[tIndex].SetSelected(true); + bin.second[tIndex].SetTracking(true); + _selectedPts.push_back(bin.second[tIndex]); + + ret.push_back(bin.second[tIndex].pt); + } + } + return ret; +} + +std::vector TrackingPointSelector::GetTrackedFeatures() +{ + std::vector selectedPoints; + for (std::vector::iterator it = _selectedPts.begin(); it != _selectedPts.end(); ++it) { + if (it->IsTracking()) { + selectedPoints.push_back(it->pt); + } + } + return selectedPoints; +} + +std::vector TrackingPointSelector::GetTrackedFeatures3d() +{ + std::vector selectedPoints; + for (std::vector::iterator it = _selectedPts.begin(); it != _selectedPts.end(); ++it) { + if (it->IsTracking()) { + selectedPoints.push_back(it->pt3d); + } + } + return selectedPoints; +} + +std::vector TrackingPointSelector::GetTrackedFeaturesWarped() +{ + std::vector selectedPoints = GetTrackedFeatures(); + std::vector warpedPoints; + perspectiveTransform(selectedPoints, warpedPoints, _homography); + return warpedPoints; +} + +std::vector TrackingPointSelector::GetAllFeatures() +{ + std::vector allBinnedPoints; + for (auto &track : trackingPointBin) { + for (auto &trackPt : track.second) { + allBinnedPoints.push_back(trackPt.pt); + } + } + return allBinnedPoints; +} + +void TrackingPointSelector::CleanUp() +{ + _selectedPts.clear(); + _pts.clear(); + trackingPointBin.clear(); + _homography.release(); +} diff --git a/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/WebARKitConfig.cpp b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/WebARKitConfig.cpp index 3e5657d..82072b8 100644 --- a/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/WebARKitConfig.cpp +++ b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/WebARKitConfig.cpp @@ -6,6 +6,7 @@ extern const int DEFAULT_MAX_FEATURES = 8000; extern const int TEBLID_MAX_FEATURES = 10000; extern const int N = 10; extern const int MIN_NUM_MATCHES = 8; +extern const int markerTemplateWidth = 15; ///< Width in pixels of image patches used in template matching. extern const int maxLevel = 3; ///< Maximum number of levels in optical flow image pyramid. extern const cv::Size winSize(31, 31); extern const cv::TermCriteria termcrit(cv::TermCriteria::COUNT | cv::TermCriteria::EPS, 20, 0.03); @@ -14,6 +15,7 @@ extern const double featureDetectPyramidLevel = extern const int featureBorder = 8; extern const cv::Size blurSize(3, 3); extern const double ransac_thresh = 2.5f; +extern cv::RNG rng( 0xFFFFFFFF ); extern const double m_pi = 3.14159265358979323846; extern const std::string WEBARKIT_HEADER_VERSION_STRING = "1.0.0"; /*@ diff --git a/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/WebARKitTracker.cpp b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/WebARKitTracker.cpp index 9e6af98..b48b3f5 100644 --- a/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/WebARKitTracker.cpp +++ b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/WebARKitTracker.cpp @@ -1,14 +1,15 @@ #include -#include #include +#include +#include namespace webarkit { class WebARKitTracker::WebARKitTrackerImpl { public: WebARKitTrackerImpl() - : corners(4), initialized(false), output(17, 0.0), _valid(false), _isDetected(false), _isTracking(false), numMatches(0), - minNumMatches(MIN_NUM_MATCHES), _nn_match_ratio(0.7f) { + : corners(4), initialized(false), output(17, 0.0), _valid(false), _isDetected(false), _isTracking(false), + numMatches(0), minNumMatches(MIN_NUM_MATCHES), _nn_match_ratio(0.7f) { m_camMatrix = cv::Matx33d::zeros(); m_distortionCoeff = cv::Mat::zeros(4, 1, cv::DataType::type); }; @@ -53,15 +54,14 @@ class WebARKitTracker::WebARKitTrackerImpl { _prevPyramid.clear(); } - template - void initTracker(T refData, size_t refCols, size_t refRows, ColorSpace colorSpace) { + template void initTracker(T refData, size_t refCols, size_t refRows, ColorSpace colorSpace) { WEBARKIT_LOGi("Init Tracker!\n"); cv::Mat refGray = convert2Grayscale(refData, refCols, refRows, colorSpace); cv::Mat trackerFeatureMask = createTrackerFeatureMask(refGray); - if(!extractFeatures(refGray, trackerFeatureMask, refKeyPts, refDescr)) { + if (!extractFeatures(refGray, trackerFeatureMask, refKeyPts, refDescr)) { WEBARKIT_LOGe("No features detected!\n"); return; }; @@ -99,17 +99,20 @@ class WebARKitTracker::WebARKitTrackerImpl { _bBox.push_back(cv::Point2f(refCols, refRows)); _bBox.push_back(cv::Point2f(0, refRows)); + _trackSelection = TrackingPointSelector(Points(refKeyPts), refCols, refRows, markerTemplateWidth); + initialized = true; WEBARKIT_LOGi("Tracker ready!\n"); } - bool extractFeatures(const cv::Mat& grayImage, cv::Mat& featureMask, std::vector& keypoints, cv::Mat& descriptors) const { + bool extractFeatures(const cv::Mat& grayImage, cv::Mat& featureMask, std::vector& keypoints, + cv::Mat& descriptors) const { assert(!grayImage.empty()); assert(grayImage.channels() == 1); this->_featureDetector->detect(grayImage, keypoints, featureMask); - if (keypoints.empty()){ + if (keypoints.empty()) { WEBARKIT_LOGe("No keypoints detected!\n"); return false; } @@ -119,15 +122,16 @@ class WebARKitTracker::WebARKitTrackerImpl { return false; } return true; - } + } - void processFrameData(uchar* frameData, size_t frameCols, size_t frameRows, ColorSpace colorSpace, bool enableBlur) { + void processFrameData(uchar* frameData, size_t frameCols, size_t frameRows, ColorSpace colorSpace, + bool enableBlur) { cv::Mat grayFrame = convert2Grayscale(frameData, frameCols, frameRows, colorSpace); if (enableBlur) { cv::blur(grayFrame, grayFrame, blurSize); - } - buildImagePyramid(grayFrame); - processFrame(grayFrame); + } + //buildImagePyramid(grayFrame); + processFrame2(grayFrame); grayFrame.release(); }; @@ -160,7 +164,7 @@ class WebARKitTracker::WebARKitTrackerImpl { cv::Mat featureMask = createFeatureMask(currIm); - if(!extractFeatures(currIm, featureMask, frameKeyPts, frameDescr)) { + if (!extractFeatures(currIm, featureMask, frameKeyPts, frameDescr)) { WEBARKIT_LOGe("No features detected in resetTracking!\n"); return false; }; @@ -174,11 +178,11 @@ class WebARKitTracker::WebARKitTrackerImpl { WEBARKIT_LOG("Num Matches: %d\n", numMatches); if (numMatches >= minNumMatches) { - //m_H = cv::findHomography(refPts, framePts, cv::RANSAC); + // m_H = cv::findHomography(refPts, framePts, cv::RANSAC); webarkit::homography::WebARKitHomographyInfo homographyInfo = getHomographyInliers(refPts, framePts); valid = homographyInfo.validHomography; - - //if ((valid = homographyValid(m_H))) { + + // if ((valid = homographyValid(m_H))) { if (valid) { m_H = homographyInfo.homography; _isDetected = true; @@ -201,14 +205,14 @@ class WebARKitTracker::WebARKitTrackerImpl { // use optical flow to track keypoints std::vector err; - //std::vector status; + // std::vector status; std::vector statusFirstPass, statusSecondPass; std::vector currPts, goodPtsCurr, goodPtsPrev; bool valid; - calcOpticalFlowPyrLK(_prevPyramid, _pyramid, framePts, currPts, statusFirstPass, err, winSize, maxLevel, termcrit, 0, - 0.001); - calcOpticalFlowPyrLK(_pyramid, _prevPyramid, currPts, framePts, statusSecondPass, err, winSize, maxLevel, termcrit, 0, - 0.001); + calcOpticalFlowPyrLK(_prevPyramid, _pyramid, framePts, currPts, statusFirstPass, err, winSize, maxLevel, + termcrit, 0, 0.001); + calcOpticalFlowPyrLK(_pyramid, _prevPyramid, currPts, framePts, statusSecondPass, err, winSize, maxLevel, + termcrit, 0, 0.001); // calculate average variance double mean, avg_variance = 0.0; double sum = 0.0; @@ -227,8 +231,8 @@ class WebARKitTracker::WebARKitTrackerImpl { goodPtsPrev.push_back(framePts[j]); diff = sqrt(pow(currPts[j].x - framePts[j].x, 2.0) + pow(currPts[j].y - framePts[j].y, 2.0)); - sum += diff; - diffs.push_back(diff); + sum += diff; + diffs.push_back(diff); } mean = sum / diffs.size(); @@ -246,10 +250,10 @@ class WebARKitTracker::WebARKitTrackerImpl { transform.push_back(row); // update homography matrix - if(!m_H.empty()) { - m_H = transform * m_H; + if (!m_H.empty()) { + m_H = transform * m_H; } - //m_H = transform * m_H; + // m_H = transform * m_H; // set old points to new points framePts = goodPtsCurr; @@ -283,41 +287,122 @@ class WebARKitTracker::WebARKitTrackerImpl { WEBARKIT_LOGi("Start tracking!\n"); clear_output(); - std::cout << _prevPyramid.size() << std::endl; - + // std::cout << _prevPyramid.size() << std::endl; if (_prevPyramid.size() > 0) { - //std::cout << "Starting Optical Flow" << std::endl; - std::vector warpedPoints; - perspectiveTransform(framePts, warpedPoints, m_H); - //if (!runOpticalFlow(i, trackablePoints, trackablePointsWarped)) { - if (!runOpticalFlow(i, framePts, warpedPoints)) { - //std::cout << "Optical flow failed." << std::endl; - return false; - } else { - //if (_trackVizActive) _trackViz.opticalFlowOK = true; - // Refine optical flow with template match. - /*if (!RunTemplateMatching(frame, i)) { - //std::cout << "Template matching failed." << std::endl; - }*/ - return true; - } - } - + // std::cout << "Starting Optical Flow" << std::endl; + std::vector warpedPoints; + // perspectiveTransform(framePts, warpedPoints, m_H); + warpedPoints = getSelectedFeaturesWarped(m_H); + // if (!runOpticalFlow(i, trackablePoints, trackablePointsWarped)) { + if (!runOpticalFlow(i, framePts, warpedPoints)) { + std::cout << "Optical flow failed." << std::endl; + return true; + } else { + // if (_trackVizActive) _trackViz.opticalFlowOK = true; + // Refine optical flow with template match. + /*if (!RunTemplateMatching(frame, i)) { + //std::cout << "Template matching failed." << std::endl; + }*/ + // std::cout << "Optical flow ok." << std::endl; + return false; + } + } } void processFrame(cv::Mat& frame) { if (!this->_valid) { this->_valid = resetTracking(frame); } else { - if(!track2()) { - + if (!track2()) { }; } WEBARKIT_LOG("Marker detected : %s\n", _isDetected ? "true" : "false"); swapImagePyramid(); }; + void processFrame2(cv::Mat& frame) { + buildImagePyramid(frame); + + if (!initialized) { + WEBARKIT_LOGe("Reference image not found. AR is unintialized!\n"); + assert(initialized == false); + } + + WEBARKIT_LOGi("Reset Tracking!\n"); + + clear_output(); + + _isDetected = false; + + cv::Mat frameDescr; + std::vector frameKeyPts; + bool valid; + + cv::Mat featureMask = createFeatureMask(frame); + + if (!extractFeatures(frame, featureMask, frameKeyPts, frameDescr)) { + WEBARKIT_LOGe("No features detected in resetTracking!\n"); + // return false; + }; + if (!_isDetected) { + std::vector refPts; + + getMatches(frameDescr, frameKeyPts, refPts, framePts); + numMatches = framePts.size(); + + WEBARKIT_LOG("Num Matches: %d\n", numMatches); + + if (numMatches >= minNumMatches) { + // m_H = cv::findHomography(refPts, framePts, cv::RANSAC); + webarkit::homography::WebARKitHomographyInfo homographyInfo = getHomographyInliers(refPts, framePts); + valid = homographyInfo.validHomography; + + // if ((valid = homographyValid(m_H))) { + if (valid) { + m_H = homographyInfo.homography; + _isDetected = true; + _trackSelection.ResetSelection(); + _trackSelection.SetHomography(homographyInfo.homography); + perspectiveTransform(_bBox, _bBoxTransformed, homographyInfo.homography); + } + } + } + int i = 0; + if (_isDetected) { + WEBARKIT_LOGi("Start tracking!\n"); + if (_prevPyramid.size() > 0) { + // std::cout << "Starting Optical Flow" << std::endl; + //std::vector warpedPoints; + // perspectiveTransform(framePts, warpedPoints, m_H); + //warpedPoints = getSelectedFeaturesWarped(m_H); + std::vector trackablePoints = _trackSelection.GetInitialFeatures(); + std::vector trackablePointsWarped = _trackSelection.GetTrackedFeaturesWarped(); + if (!runOpticalFlow(i, trackablePoints, trackablePointsWarped)) { + //if (!runOpticalFlow(i, framePts, warpedPoints)) { + std::cout << "Optical flow failed." << std::endl; + // return true; + } else { + // if (_trackVizActive) _trackViz.opticalFlowOK = true; + // Refine optical flow with template match. + /*if (!RunTemplateMatching(frame, i)) { + //std::cout << "Template matching failed." << std::endl; + }*/ + // std::cout << "Optical flow ok." << std::endl; + // return false; + } + } + } + + if (_isDetected || _isTracking) { + // std::vector warpedCorners = _trackSelection.GetTrackedFeaturesWarped(); + // _patternTrackingInfo.computePose(_pattern.points3d, warpedCorners, m_camMatrix, m_distortionCoeff); + } + + WEBARKIT_LOG("Marker detected : %s\n", _isDetected ? "true" : "false"); + swapImagePyramid(); + } + bool homographyValid(cv::Mat& H) { if (H.empty()) { return false; @@ -350,25 +435,23 @@ class WebARKitTracker::WebARKitTrackerImpl { output[16] = warped[3].y; }; - void clear_output() { - output = std::vector(17, 0.0); - }; + void clear_output() { output = std::vector(17, 0.0); }; - void buildImagePyramid(cv::Mat& frame) { - cv::buildOpticalFlowPyramid(frame, _pyramid, winSize, maxLevel); - } + void buildImagePyramid(cv::Mat& frame) { cv::buildOpticalFlowPyramid(frame, _pyramid, winSize, maxLevel); } void swapImagePyramid() { _pyramid.swap(_prevPyramid); } - bool runOpticalFlow(int trackableId, const std::vector& trackablePoints, const std::vector& trackablePointsWarped) - { + bool runOpticalFlow(int trackableId, const std::vector& trackablePoints, + const std::vector& trackablePointsWarped) { std::vector flowResultPoints, trackablePointsWarpedResult; std::vector statusFirstPass, statusSecondPass; std::vector err; - cv::calcOpticalFlowPyrLK(_prevPyramid, _pyramid, trackablePointsWarped, flowResultPoints, statusFirstPass, err, winSize, maxLevel, termcrit, 0, 0.001); + cv::calcOpticalFlowPyrLK(_prevPyramid, _pyramid, trackablePointsWarped, flowResultPoints, statusFirstPass, err, + winSize, maxLevel, termcrit, 0, 0.001); // By using bi-directional optical flow, we improve quality of detected points. - cv::calcOpticalFlowPyrLK(_pyramid, _prevPyramid, flowResultPoints, trackablePointsWarpedResult, statusSecondPass, err, winSize, maxLevel, termcrit, 0, 0.001); - + cv::calcOpticalFlowPyrLK(_pyramid, _prevPyramid, flowResultPoints, trackablePointsWarpedResult, + statusSecondPass, err, winSize, maxLevel, termcrit, 0, 0.001); + // Keep only the points for which flow was found in both temporal directions. int killed1 = 0; std::vector filteredTrackablePoints, filteredTrackedPoints; @@ -381,11 +464,13 @@ class WebARKitTracker::WebARKitTrackerImpl { filteredTrackablePoints.push_back(trackablePoints[j]); filteredTrackedPoints.push_back(flowResultPoints[j]); } + // std::cout << "Optical Flow ok!!!!" << std::endl; /*if (_trackVizActive) { _trackViz.opticalFlowTrackablePoints = filteredTrackablePoints; _trackViz.opticalFlowTrackedPoints = filteredTrackedPoints; }*/ - //std::cout << "Optical flow discarded " << killed1 << " of " << flowResultPoints.size() << " points" << std::endl; + // std::cout << "Optical flow discarded " << killed1 << " of " << flowResultPoints.size() << " points" << + // std::endl; if (!updateTrackableHomography(trackableId, filteredTrackablePoints, filteredTrackedPoints)) { _isDetected = false; @@ -398,17 +483,17 @@ class WebARKitTracker::WebARKitTrackerImpl { return true; } - bool updateTrackableHomography(int trackableId, const std::vector& matchedPoints1, const std::vector& matchedPoints2) - { + bool updateTrackableHomography(int trackableId, const std::vector& matchedPoints1, + const std::vector& matchedPoints2) { if (matchedPoints1.size() > 4) { homography::WebARKitHomographyInfo homoInfo = getHomographyInliers(matchedPoints1, matchedPoints2); if (homoInfo.validHomography) { - //_trackables[trackableId]._trackSelection.UpdatePointStatus(homoInfo.status); - //_trackables[trackableId]._trackSelection.SetHomography(homoInfo.homography); + _trackSelection.UpdatePointStatus(homoInfo.status); + _trackSelection.SetHomography(homoInfo.homography); m_H = homoInfo.homography; this->_valid = homoInfo.validHomography; // Update the bounding box. - //perspectiveTransform(_bBox, _bBoxTransformed, homoInfo.homography); + // perspectiveTransform(_bBox, _bBoxTransformed, homoInfo.homography); fill_output(m_H); /*if (_trackVizActive) { for (int i = 0; i < 4; i++) { @@ -425,7 +510,8 @@ class WebARKitTracker::WebARKitTrackerImpl { return false; } - void getMatches(const cv::Mat& frameDescr, std::vector& frameKeyPts, std::vector& refPoints, std::vector& framePoints) { + void getMatches(const cv::Mat& frameDescr, std::vector& frameKeyPts, + std::vector& refPoints, std::vector& framePoints) { std::vector> knnMatches; _matcher->knnMatch(frameDescr, refDescr, knnMatches, 2); @@ -502,6 +588,8 @@ class WebARKitTracker::WebARKitTrackerImpl { WebARKitPatternTrackingInfo _patternTrackingInfo; + TrackingPointSelector _trackSelection; + cv::Matx33d m_camMatrix; cv::Mat m_distortionCoeff; @@ -569,7 +657,8 @@ void WebARKitTracker::initTracker(uchar* refData, size_t refCols, size_t refRows _trackerImpl->initTracker(refData, refCols, refRows, colorSpace); } -void WebARKitTracker::processFrameData(uchar* frameData, size_t frameCols, size_t frameRows, ColorSpace colorSpace, bool enableBlur) { +void WebARKitTracker::processFrameData(uchar* frameData, size_t frameCols, size_t frameRows, ColorSpace colorSpace, + bool enableBlur) { _trackerImpl->processFrameData(frameData, frameCols, frameRows, colorSpace, enableBlur); } diff --git a/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/TrackedPoint.h b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/TrackedPoint.h new file mode 100644 index 0000000..8d6965c --- /dev/null +++ b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/TrackedPoint.h @@ -0,0 +1,61 @@ +/* + * TrackedPoint.h + * artoolkitX + * + * This file is part of artoolkitX. + * + * artoolkitX is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * artoolkitX is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with artoolkitX. If not, see . + * + * As a special exception, the copyright holders of this library give you + * permission to link this library with independent modules to produce an + * executable, regardless of the license terms of these independent modules, and to + * copy and distribute the resulting executable under terms of your choice, + * provided that you also meet, for each linked independent module, the terms and + * conditions of the license of that module. An independent module is a module + * which is neither derived from nor based on this library. If you modify this + * library, you may extend this exception to your version of the library, but you + * are not obligated to do so. If you do not wish to do so, delete this exception + * statement from your version. + * + * Copyright 2018 Realmax, Inc. + * Copyright 2015 Daqri, LLC. + * Copyright 2010-2015 ARToolworks, Inc. + * + * Author(s): Philip Lamb, Daniel Bell. + * Modified for WebARKit by @kalwalt - Walter Perdan - 2024 + * + */ + +#ifndef TRACKED_POINT_H +#define TRACKED_POINT_H + +#include + +class TrackedPoint +{ +public: + int id; + cv::Point2f pt; + cv::Point3f pt3d; + cv::Rect markerRoi; + bool selected; + bool tracking; + + bool IsTracking(); + void SetTracking(bool newTracking); + bool IsSelected(); + void SetSelected(bool newSelected); +}; + +#endif //TRACKED_POINT diff --git a/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/TrackingPointSelector.h b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/TrackingPointSelector.h new file mode 100644 index 0000000..c6f0094 --- /dev/null +++ b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/TrackingPointSelector.h @@ -0,0 +1,94 @@ +/* + * TrackingPointSelector.h + * artoolkitX + * + * This file is part of artoolkitX. + * + * artoolkitX is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * artoolkitX is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with artoolkitX. If not, see . + * + * As a special exception, the copyright holders of this library give you + * permission to link this library with independent modules to produce an + * executable, regardless of the license terms of these independent modules, and to + * copy and distribute the resulting executable under terms of your choice, + * provided that you also meet, for each linked independent module, the terms and + * conditions of the license of that module. An independent module is a module + * which is neither derived from nor based on this library. If you modify this + * library, you may extend this exception to your version of the library, but you + * are not obligated to do so. If you do not wish to do so, delete this exception + * statement from your version. + * + * Copyright 2018 Realmax, Inc. + * Copyright 2015 Daqri, LLC. + * Copyright 2010-2015 ARToolworks, Inc. + * + * Author(s): Philip Lamb, Daniel Bell. + * Modified for WebARKit by @kalwalt - Walter Perdan - 2024 + * + */ + +#ifndef TRACKINGPOINTSELECTOR_H +#define TRACKINGPOINTSELECTOR_H +#include +#include +#include +#include + +/** + @brief Class used to manage selection of tracking points based on image templates (i.e. unique pixel patches). + */ +class TrackingPointSelector +{ +public: + TrackingPointSelector(); + + TrackingPointSelector(std::vector pts, int width, int height, int markerTemplateWidth); + + void DistributeBins(int width, int height, int markerTemplateWidth); + + void SetHomography(cv::Mat newHomography); + + cv::Mat GetHomography(); + + void UpdatePointStatus(std::vector status); + + /** + @brief Signal that the next call to GetInitialFeatures should return a new selection. + */ + void ResetSelection(); + + /** + @brief If reset, then selects an initial random template from each bin for tracking, + and returns this set. If not reset then returns the same set as GetTrackedFeatures. + */ + std::vector GetInitialFeatures(); + + std::vector GetTrackedFeatures(); + + std::vector GetTrackedFeatures3d(); + + std::vector GetTrackedFeaturesWarped(); + + /// Get all points from all bins that are candidates for selection. + std::vector GetAllFeatures(); + + void CleanUp(); + +private: + bool _reset; + std::vector _pts; + std::map > trackingPointBin; + cv::Mat _homography; + std::vector _selectedPts; +}; +#endif //TRACKINGPOINTSELECTOR diff --git a/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/WebARKitConfig.h b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/WebARKitConfig.h index f330d10..584d078 100644 --- a/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/WebARKitConfig.h +++ b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/WebARKitConfig.h @@ -14,13 +14,15 @@ extern const int DEFAULT_MAX_FEATURES; extern const int TEBLID_MAX_FEATURES; extern const int N; extern const int MIN_NUM_MATCHES; +extern const int markerTemplateWidth; ///< Width in pixels of image patches used in template matching. extern const int maxLevel; ///< Maximum number of levels in optical flow image pyramid. extern const cv::Size winSize; extern const cv::TermCriteria termcrit; extern const double featureDetectPyramidLevel; ///> Scale factor applied to image pyramid to determine image to perform feature matching upon. extern const int featureBorder; extern const cv::Size blurSize; -extern const double ransac_thresh; +extern const double ransac_thresh; +extern cv::RNG rng; extern const double m_pi; extern const std::string WEBARKIT_HEADER_VERSION_STRING; extern const int WEBARKIT_HEADER_VERSION_MAJOR; diff --git a/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/WebARKitUtils.h b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/WebARKitUtils.h index 492239c..4debaeb 100644 --- a/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/WebARKitUtils.h +++ b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/WebARKitUtils.h @@ -1,45 +1,58 @@ #ifndef WEBARKIT_UTILS_H #define WEBARKIT_UTILS_H -//#include +// #include #include #include #include namespace webarkit { +static std::vector Points(std::vector keypoints) { + std::vector res; + for (unsigned i = 0; i < keypoints.size(); i++) { + res.push_back(keypoints[i].pt); + } + return res; +} + /// Method for calculating and validating a homography matrix from a set of corresponding points. /// pts1 and pts must have the same dimensionality. -/// @returns An WebARKitHomographyInfo instance, with its status vector of the same dimensionality as the pts1 and pts2 vectors. -static homography::WebARKitHomographyInfo getHomographyInliers(std::vector pts1, std::vector pts2) -{ +/// @returns An WebARKitHomographyInfo instance, with its status vector of the same dimensionality as the pts1 and pts2 +/// vectors. +static homography::WebARKitHomographyInfo getHomographyInliers(std::vector pts1, + std::vector pts2) { if (pts1.size() < 4) { return homography::WebARKitHomographyInfo(); } - + cv::Mat inlier_mask, homography; homography = findHomography(pts1, pts2, cv::RANSAC, ransac_thresh, inlier_mask); if (homography.empty()) { // Failed to find a homography. return homography::WebARKitHomographyInfo(); } - - const double det = homography.at(0, 0) * homography.at(1, 1) - homography.at(1, 0) * homography.at(0, 1); + + const double det = homography.at(0, 0) * homography.at(1, 1) - + homography.at(1, 0) * homography.at(0, 1); if (det < 0) { return homography::WebARKitHomographyInfo(); } - - const double N1 = sqrt(homography.at(0, 0) * homography.at(0, 0) + homography.at(1, 0) * homography.at(1, 0)); + + const double N1 = sqrt(homography.at(0, 0) * homography.at(0, 0) + + homography.at(1, 0) * homography.at(1, 0)); if (N1 > 4 || N1 < 0.1) { return homography::WebARKitHomographyInfo(); } - - const double N2 = sqrt(homography.at(0, 1) * homography.at(0, 1) + homography.at(1, 1) * homography.at(1, 1)); + + const double N2 = sqrt(homography.at(0, 1) * homography.at(0, 1) + + homography.at(1, 1) * homography.at(1, 1)); if (N2 > 4 || N2 < 0.1) { return homography::WebARKitHomographyInfo(); } - - const double N3 = sqrt(homography.at(2, 0) * homography.at(2, 0) + homography.at(2, 1) * homography.at(2, 1)); + + const double N3 = sqrt(homography.at(2, 0) * homography.at(2, 0) + + homography.at(2, 1) * homography.at(2, 1)); if (N3 > 0.002) { return homography::WebARKitHomographyInfo(); } @@ -48,7 +61,7 @@ static homography::WebARKitHomographyInfo getHomographyInliers(std::vector inlier_matches; int linliers = 0; for (int i = 0; i < pts1.size(); i++) { - if ((int)inlier_mask.at(i,0) == 1) { + if ((int)inlier_mask.at(i, 0) == 1) { status.push_back((uchar)1); inlier_matches.push_back(cv::DMatch(i, i, 0)); linliers++; @@ -56,7 +69,7 @@ static homography::WebARKitHomographyInfo getHomographyInliers(std::vector