Improving c++ code

.github/workflows/test.yml

@@ -5,11 +5,11 @@ jobs:
     build:
         runs-on: ubuntu-latest
         steps:
-        - uses: actions/checkout@v3
+        - uses: actions/checkout@v4
         - name: Setup Node.js
-          uses: actions/setup-node@v3
+          uses: actions/setup-node@v4
           with:
-            node-version: '16.x'
+            node-version: '20.x'
         - name: Build and test WebARKitLib
           run: |
               cd tests && mkdir build && cd build && cmake -DEMSCRIPTEN_COMP=0 .. && make && ./webarkit_test

WebARKit/CMakeLists.txt

@@ -29,8 +29,11 @@ FetchContent_MakeAvailable(build_opencv)
 get_filename_component(PARENT_DIR ./ ABSOLUTE)
 
 set(WEBARKIT_HEADERS
+${PARENT_DIR}/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/TrackedPoint.h
+${PARENT_DIR}/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/TrackingPointSelector.h
 ${PARENT_DIR}/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/WebARKitConfig.h
-${PARENT_DIR}/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking//WebARKitEnums.h
+${PARENT_DIR}/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/WebARKitEnums.h
+${PARENT_DIR}/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/WebARKitHomographyInfo.h
 ${PARENT_DIR}/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/WebARKitTracker.h
 ${PARENT_DIR}/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/WebARKitUtils.h
 ${PARENT_DIR}/include/WebARKitCamera.h
@@ -41,7 +44,10 @@ ${PARENT_DIR}/include/WebARKitPattern.h
 )
 
 set(SOURCE 
+${PARENT_DIR}/WebARKitTrackers/WebARKitOpticalTracking/TrackedPoint.cpp
+${PARENT_DIR}/WebARKitTrackers/WebARKitOpticalTracking/TrackingPointSelector.cpp
 ${PARENT_DIR}/WebARKitTrackers/WebARKitOpticalTracking/WebARKitConfig.cpp
+${PARENT_DIR}/WebARKitTrackers/WebARKitOpticalTracking/WebARKitHomographyInfo.cpp
 ${PARENT_DIR}/WebARKitTrackers/WebARKitOpticalTracking/WebARKitTracker.cpp
 ${PARENT_DIR}/WebARKitCamera.cpp
 ${PARENT_DIR}/WebARKitLog.cpp

WebARKit/WebARKitManager.cpp

@@ -101,6 +101,10 @@ cv::Mat WebARKitManager::getPoseMatrix() {
     return m_tracker->getPoseMatrix();
 }
 
+float* WebARKitManager::getPoseMatrix2() {
+    return m_tracker->getPoseMatrix2();
+}
+
 cv::Mat WebARKitManager::getGLViewMatrix() {
     return m_tracker->getGLViewMatrix();
 }

WebARKit/WebARKitPattern.cpp

@@ -7,18 +7,32 @@ WebARKitPatternTrackingInfo::WebARKitPatternTrackingInfo() {
     m_scale = 1.0f;
 }
 
+void WebARKitPatternTrackingInfo::cameraPoseFromPoints(cv::Mat& pose, const std::vector<cv::Point3f>& objPts,
+                                                       const std::vector<cv::Point2f>& imgPts, const cv::Matx33f& caMatrix,
+                                              const cv::Mat& distCoeffs) {
+    cv::Mat rvec = cv::Mat::zeros(3, 1, CV_64FC1); // output rotation vector
+    cv::Mat tvec = cv::Mat::zeros(3, 1, CV_64FC1); // output translation vector
+
+    cv::solvePnPRansac(objPts, imgPts, caMatrix, distCoeffs, rvec, tvec);
+
+    // Assemble pose matrix from rotation and translation vectors.
+    cv::Mat rMat;
+    Rodrigues(rvec, rMat);
+    cv::hconcat(rMat, tvec, pose);
+};
+
 void WebARKitPatternTrackingInfo::computePose(std::vector<cv::Point3f>& treeDPoints,
                                               std::vector<cv::Point2f>& imgPoints, const cv::Matx33f& caMatrix,
                                               const cv::Mat& distCoeffs) {
-    //cv::Mat rvec = cv::Mat::zeros(3, 1, CV_64FC1); // output rotation vector
-    //cv::Mat tvec = cv::Mat::zeros(3, 1, CV_64FC1); // output translation vector
+    // cv::Mat rvec = cv::Mat::zeros(3, 1, CV_64FC1); // output rotation vector
+    // cv::Mat tvec = cv::Mat::zeros(3, 1, CV_64FC1); // output translation vector
     cv::Mat rvec, tvec;
 
     cv::solvePnPRansac(treeDPoints, imgPoints, caMatrix, distCoeffs, rvec, tvec);
 
     cv::Mat rMat;
     cv::Rodrigues(rvec, rMat);
-    //cv::hconcat(rMat, tvec, pose3d);
+    // cv::hconcat(rMat, tvec, pose3d);
 
     for (unsigned int row = 0; row < 3; ++row) {
         for (unsigned int col = 0; col < 3; ++col) {
@@ -31,10 +45,16 @@ void WebARKitPatternTrackingInfo::computePose(std::vector<cv::Point3f>& treeDPoi
     invertPose();
 }
 
-void WebARKitPatternTrackingInfo::computeGLviewMatrix() {
-   cv::transpose(pose3d , glViewMatrix);
+void WebARKitPatternTrackingInfo::getTrackablePose(cv::Mat& pose) {
+    //float transMat [3][4];
+    cv::Mat poseOut;
+    pose.convertTo(poseOut, CV_32FC1);
+    //std::cout << "poseOut: " << poseOut << std::endl;
+    memcpy(transMat, poseOut.ptr<float>(0), 3*4*sizeof(float));
 }
 
+void WebARKitPatternTrackingInfo::computeGLviewMatrix() { cv::transpose(pose3d, glViewMatrix); }
+
 void WebARKitPatternTrackingInfo::invertPose() {
 
     /*cv::Mat invertPose(3, 4, CV_64FC1);

WebARKit/WebARKitTrackers/WebARKitOpticalTracking/TrackedPoint.cpp

@@ -0,0 +1,60 @@
+/*
+ *  TrackedPoint.cpp
+ *  artoolkitX
+ *
+ *  This file is part of artoolkitX.
+ *
+ *  artoolkitX is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU Lesser General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  artoolkitX is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Lesser General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Lesser General Public License
+ *  along with artoolkitX.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *  As a special exception, the copyright holders of this library give you
+ *  permission to link this library with independent modules to produce an
+ *  executable, regardless of the license terms of these independent modules, and to
+ *  copy and distribute the resulting executable under terms of your choice,
+ *  provided that you also meet, for each linked independent module, the terms and
+ *  conditions of the license of that module. An independent module is a module
+ *  which is neither derived from nor based on this library. If you modify this
+ *  library, you may extend this exception to your version of the library, but you
+ *  are not obligated to do so. If you do not wish to do so, delete this exception
+ *  statement from your version.
+ *
+ *  Copyright 2018 Realmax, Inc.
+ *  Copyright 2015 Daqri, LLC.
+ *  Copyright 2010-2015 ARToolworks, Inc.
+ *
+ *  Author(s): Philip Lamb, Daniel Bell.
+ *  Modified for WebARKit by @kalwalt - Walter Perdan - 2024
+ *
+ */
+
+#include <WebARKitTrackers/WebARKitOpticalTracking/TrackedPoint.h>
+
+bool TrackedPoint::IsTracking()
+{
+    return tracking;
+}
+
+void TrackedPoint::SetTracking(bool newTracking)
+{
+    tracking = newTracking;
+}
+
+bool TrackedPoint::IsSelected()
+{
+    return selected;
+}
+
+void TrackedPoint::SetSelected(bool newSelected)
+{
+    selected = newSelected;
+}

WebARKit/WebARKitTrackers/WebARKitOpticalTracking/TrackingPointSelector.cpp

@@ -0,0 +1,194 @@
+/*
+ *  TrackingPointSelector.cpp
+ *  artoolkitX
+ *
+ *  This file is part of artoolkitX.
+ *
+ *  artoolkitX is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU Lesser General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  artoolkitX is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Lesser General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Lesser General Public License
+ *  along with artoolkitX.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *  As a special exception, the copyright holders of this library give you
+ *  permission to link this library with independent modules to produce an
+ *  executable, regardless of the license terms of these independent modules, and to
+ *  copy and distribute the resulting executable under terms of your choice,
+ *  provided that you also meet, for each linked independent module, the terms and
+ *  conditions of the license of that module. An independent module is a module
+ *  which is neither derived from nor based on this library. If you modify this
+ *  library, you may extend this exception to your version of the library, but you
+ *  are not obligated to do so. If you do not wish to do so, delete this exception
+ *  statement from your version.
+ *
+ *  Copyright 2018 Realmax, Inc.
+ *  Copyright 2015 Daqri, LLC.
+ *  Copyright 2010-2015 ARToolworks, Inc.
+ *
+ *  Author(s): Philip Lamb, Daniel Bell.
+ *  Modified for WebARKit by @kalwalt - Walter Perdan - 2024
+ *
+ */
+
+#include <WebARKitTrackers/WebARKitOpticalTracking/TrackingPointSelector.h>
+
+TrackingPointSelector::TrackingPointSelector()
+{
+}
+
+TrackingPointSelector::TrackingPointSelector(std::vector<cv::Point2f> pts, int width, int height, int markerTemplateWidth) :
+    _reset(false),
+    _pts(pts)
+{
+    DistributeBins(width, height, markerTemplateWidth);
+}
+
+/**
+    @brief Iterates over \_pts and for each one, provided it doesn't intersect the image border,
+    creates a TrackedPoint representing a template () with a serially-increasing id from 0, and puts
+    it into the trackingPointsBin structure (a vector of pairs of (binIndex, trackingPoint).
+ */
+void TrackingPointSelector::DistributeBins(int width, int height, int markerTemplateWidth)
+{
+    int numberOfBins = 10;
+
+    // Split width and height dimensions into 10 bins each, for total of 100 bins.
+    int totalXBins = width/numberOfBins;
+    int totalYBins = height/numberOfBins;
+    // Init empty bins.
+    for (int i = 0; i < (numberOfBins * numberOfBins); i++) {
+        trackingPointBin.insert(std::pair<int, std::vector<TrackedPoint> >(i, std::vector<TrackedPoint>()));
+    }
+
+    // Iterate the points and add points to each bin.
+    for (int i = 0, id = 0; i < _pts.size(); i++) {
+        int bx = (int)_pts[i].x/totalXBins;
+        int by = (int)_pts[i].y/totalYBins;
+        int index = bx + (by * numberOfBins);
+
+        cv::Rect templateRoi = cv::Rect(_pts[i].x - markerTemplateWidth, _pts[i].y - markerTemplateWidth, markerTemplateWidth*2, markerTemplateWidth*2);
+        bool is_inside = (templateRoi & cv::Rect(0, 0, width, height)) == templateRoi; // templateRoi must not intersect image boundary.
+        if (is_inside) {
+            TrackedPoint newPt;
+            newPt.id = id;
+            newPt.pt = _pts[i];
+            newPt.pt3d = cv::Point3f(_pts[i].x, _pts[i].y, 0);
+            newPt.markerRoi = templateRoi;
+            trackingPointBin[index].push_back(newPt);
+            id++;
+        }
+    }
+}
+
+void TrackingPointSelector::SetHomography(cv::Mat newHomography)
+{
+    _homography = newHomography;
+}
+
+/// @return 3x3 cv::Mat (of type CV_64FC1, i.e. double) containing the homography.
+cv::Mat TrackingPointSelector::GetHomography()
+{
+    return _homography;
+}
+
+void TrackingPointSelector::UpdatePointStatus(std::vector<uchar> status)
+{
+    int index = 0;
+    for (std::vector<TrackedPoint>::iterator it = _selectedPts.begin(); it != _selectedPts.end(); ++it) {
+        if (it->tracking) {
+            it->SetTracking((int)status[index++]);
+        }
+    }
+}
+
+void TrackingPointSelector::ResetSelection()
+{
+    _reset = true;
+}
+
+std::vector<cv::Point2f> TrackingPointSelector::GetInitialFeatures()
+{
+    if (!_reset) return GetTrackedFeatures();
+    _reset = false;
+
+    // Reset state of all points to not selected and not tracking.
+    _selectedPts.clear();
+    for (auto &bin : trackingPointBin) {
+        for (auto &trackPt : bin.second) {
+            trackPt.SetSelected(false);
+            trackPt.SetTracking(false);
+        }
+    }
+
+    // Selects a random template from each bin for tracking.
+    std::vector<cv::Point2f> ret;
+    for (auto &bin : trackingPointBin) {
+        size_t pointCount = bin.second.size();
+        if (pointCount > 0) { // If there are points in the bin.
+            // Select a random point from the bin.
+            int tIndex = pointCount > 1 ? rng.uniform(0, static_cast<int>(bin.second.size())) : 0;
+            bin.second[tIndex].SetSelected(true);
+            bin.second[tIndex].SetTracking(true);
+            _selectedPts.push_back(bin.second[tIndex]);
+
+            ret.push_back(bin.second[tIndex].pt);
+        }
+    }
+    return ret;
+}
+
+std::vector<cv::Point2f> TrackingPointSelector::GetTrackedFeatures()
+{
+    std::vector<cv::Point2f> selectedPoints;
+    for (std::vector<TrackedPoint>::iterator it = _selectedPts.begin(); it != _selectedPts.end(); ++it) {
+        if (it->IsTracking()) {
+            selectedPoints.push_back(it->pt);
+        }
+    }
+    return selectedPoints;
+}
+
+std::vector<cv::Point3f> TrackingPointSelector::GetTrackedFeatures3d()
+{
+    std::vector<cv::Point3f> selectedPoints;
+    for (std::vector<TrackedPoint>::iterator it = _selectedPts.begin(); it != _selectedPts.end(); ++it) {
+        if (it->IsTracking()) {
+            selectedPoints.push_back(it->pt3d);
+        }
+    }
+    return selectedPoints;
+}
+
+std::vector<cv::Point2f> TrackingPointSelector::GetTrackedFeaturesWarped()
+{
+    std::vector<cv::Point2f> selectedPoints = GetTrackedFeatures();
+    std::vector<cv::Point2f> warpedPoints;
+    perspectiveTransform(selectedPoints, warpedPoints, _homography);
+    return warpedPoints;
+}
+
+std::vector<cv::Point2f> TrackingPointSelector::GetAllFeatures()
+{
+    std::vector<cv::Point2f> allBinnedPoints;
+    for (auto &track : trackingPointBin) {
+        for (auto &trackPt : track.second) {
+            allBinnedPoints.push_back(trackPt.pt);
+        }
+    }
+    return allBinnedPoints;
+}
+
+void TrackingPointSelector::CleanUp()
+{
+    _selectedPts.clear();
+    _pts.clear();
+    trackingPointBin.clear();
+    _homography.release();
+}

WebARKit/WebARKitTrackers/WebARKitOpticalTracking/WebARKitConfig.cpp

@@ -2,17 +2,24 @@
 
 extern const double DEFAULT_NN_MATCH_RATIO = 0.7f;
 extern const double TEBLID_NN_MATCH_RATIO = 0.8f;
-extern const int DEFAULT_MAX_FEATURES = 8000;
-extern const int TEBLID_MAX_FEATURES = 10000;
+extern const int DEFAULT_MAX_FEATURES = 800;
+extern const int TEBLID_MAX_FEATURES = 1000;
 extern const int N = 10;
 extern const int MIN_NUM_MATCHES = 8;
+extern const int minRequiredDetectedFeatures = 50; ///< Minimum number of detected features required to consider a target matched.
+extern const int markerTemplateWidth = 15; ///< Width in pixels of image patches used in template matching.
 extern const int maxLevel = 3; ///< Maximum number of levels in optical flow image pyramid.
 extern const cv::Size winSize(31, 31);
 extern const cv::TermCriteria termcrit(cv::TermCriteria::COUNT | cv::TermCriteria::EPS, 20, 0.03);
+extern const int searchRadius = 15;
+extern const int match_method = cv::TM_SQDIFF_NORMED;
+extern const cv::Size featureImageMinSize(640, 480); ///< Minimum size when downscaling incoming images used for feature tracking.
 extern const double featureDetectPyramidLevel =
     1.05f; ///> Scale factor applied to image pyramid to determine image to perform feature matching upon.
 extern const int featureBorder = 8;
 extern const cv::Size blurSize(3, 3);
+extern const double ransac_thresh = 2.5f; 
+extern cv::RNG rng( 0xFFFFFFFF );
 extern const double m_pi = 3.14159265358979323846;
 extern const std::string WEBARKIT_HEADER_VERSION_STRING = "1.0.0";
 /*@