Pull Request for GSoC Work Product Submission

.gitignore

@@ -7,3 +7,5 @@ cmake_install.cmake
 install_manifest.txt
 compile_commands.json
 CTestTestfile.cmake
+CMakeLists.txt.user
+build

CMakeLists.txt

@@ -0,0 +1,88 @@
+CMAKE_MINIMUM_REQUIRED(VERSION 3.8.0)
+
+PROJECT(autocalib-sensor-extrinsics)
+
+SET(CMAKE_CXX_STANDARD 17)
+
+SET(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+
+IF(NOT CMAKE_BUILD_TYPE)
+	SET(CMAKE_BUILD_TYPE Debug)
+ENDIF()
+
+# Set the maximum level of warnings:
+SET(EXTRA_WARNINGS OFF CACHE BOOL "Enable extra warnings apart from -Wall")
+MARK_AS_ADVANCED(EXTRA_WARNINGS)
+
+# Shared options between GCC and CLANG:
+# ======================================
+IF (${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang" OR CMAKE_COMPILER_IS_GNUCXX)
+	IF(CMAKE_BUILD_TYPE MATCHES "Debug")
+		SET(EXTRA_CPP_FLAGS "${EXTRA_CPP_FLAGS} -g")
+		ADD_DEFINITIONS( -D_DEBUG)
+        	ADD_DEFINITIONS( -DDEBUG)
+	ENDIF(CMAKE_BUILD_TYPE MATCHES "Debug")
+
+        # Whole program optimization
+	IF(WHOLE_PROGRAM_OPTIMIZATION)
+	        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --combine")
+		SET(EXES_CXX_FLAGS "${EXES_CXX_FLAGS} -fwhole-program --combine")
+	ENDIF(WHOLE_PROGRAM_OPTIMIZATION)
+
+        # "-mtune=native" generates code optimized for the detected current processor.
+	IF (WIN32 OR "${CMAKE_ARCH}" MATCHES ".*86" OR "${CMAKE_ARCH}" MATCHES "amd64")
+		SET(opt_native_def ON)
+	ENDIF (WIN32 OR "${CMAKE_ARCH}" MATCHES ".*86" OR "${CMAKE_ARCH}" MATCHES "amd64")
+
+	SET( OPTIMIZE_NATIVE ${opt_native_def} CACHE BOOL "GCC/clang optimizations for current processor (-mtune=native). Requires GCC 4.2+")
+
+	IF(CMAKE_BUILD_TYPE STREQUAL "Debug")  # Not in debug!
+		SET(OPTIMIZE_NATIVE OFF)
+	ENDIF(CMAKE_BUILD_TYPE STREQUAL "Debug")
+
+        # "-ffast-math"
+	SET( OPTIMIZE_FFAST-MATH OFF CACHE BOOL "GCC/clang optimization for floating math (-ffast-math).")
+	IF(OPTIMIZE_FFAST-MATH)
+		SET(EXTRA_CPP_FLAGS "${EXTRA_CPP_FLAGS} -ffast-math")
+	ENDIF(OPTIMIZE_FFAST-MATH)
+
+	SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${EXTRA_CPP_FLAGS}")
+ENDIF ()
+
+# Project dependencies
+# ======================================
+INCLUDE(cmake_modules/script_MRPT.cmake REQUIRED)
+
+# OpenCV is required to process images from regular cameras
+FIND_PACKAGE(OpenCV REQUIRED)
+INCLUDE_DIRECTORIES(${OpenCV_INCLUDE_DIRS})
+LINK_DIRECTORIES(${OpenCV_LIBS_DIR})
+
+FIND_PACKAGE(VTK REQUIRED)
+
+# PCL is required to segment 3D planes from depth images
+FIND_PACKAGE(PCL REQUIRED)
+INCLUDE_DIRECTORIES(${PCL_INCLUDE_DIRS})
+LINK_DIRECTORIES(${PCL_LIBRARY_DIRS})
+#ADD_DEFINITIONS(${PCL_DEFINITIONS})
+
+# BOOST is required for the unit tests
+FIND_PACKAGE(Boost 1.46.0 REQUIRED system filesystem unit_test_framework serialization)
+
+# Qt5 GUI library
+FIND_PACKAGE(Qt5 COMPONENTS Widgets REQUIRED)
+
+# Set the path to this project' sources
+ADD_DEFINITIONS(-DPROJECT_SOURCE_PATH="${PROJECT_SOURCE_DIR}")
+
+INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}/core)
+INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}/gui)
+
+ADD_SUBDIRECTORY(gui)
+ADD_SUBDIRECTORY(core)
+
+#SET( BUILD_EXAMPLES ON CACHE BOOL "Build examples programs to show functions usage")
+#ADD_SUBDIRECTORY(examples)
+
+#SET( BUILD_TESTS ON CACHE BOOL "Build tests")
+#ADD_SUBDIRECTORY(test)

README.md

@@ -1 +1,120 @@
-# autocalib-sensor-extrinsics
+# Automatic Calibration of Sensor Extrinsics 
+
+## Overview
+An end-to-end application with a graphical user interface for easily calibrating the extrinsics between range and visual sensors.
+
+Specifically, the app can calibrate the extrinsics of RGB-D cameras, 3D LiDARs, and any combination between them or with RGB cameras. Automatic and target-less calibration algorithms based on plane-matching and line-matching have been integrated into the app, allowing the calibration to be performed in any generic scene setting without the need for any specific targets. The app allows visualizing the data and tuning the parameters through a GUI at each stage of the process, thereby easing calibration efforts.
+
+**Author** : [karnikram](https://github.com/karnikram)<br>
+
+**Note** : Solvers are not fully complete for real-world use, contributions are welcome!
+
+<br>
+<p align="center">
+<img src="doc/images/app-screengrab.png" width = "500"/>
+</p>
+
+### Includes 
+* A Qt-based GUI for integrating all the steps of the calibration process - raw data inspection, selecting sensors and grouping observations, feature extraction and matching, results visualization, iterative reruns with different parameters and algorithms - all into one app.
+
+* An implementation of the calibration from planes algorithm[[1]](#references) for calibrating range sensors (RGB-D cameras and 3D LiDARs), and integrated with the GUI. This includes plane extraction and matching, and least-squares minimization of the geometric cost function.
+
+* An implementation of the calibration from lines algorithm[[2]](#references) for calibration range sensors with RGB cameras, and integrated with the GUI. This includes 2D and 3D line extraction and matching, and least-squares minimization of the geometric cost function.
+
+* Well structured code using OOP principles, and with detailed documentation to encourage more contributions. The core algorithms are written independent of the GUI so they can also be potentially run from the command line, independent of the GUI.
+
+
+## Installation
+
+### Dependencies
+
+* MRPT (built from master in Debug mode)
+* Qt v5.9
+* Point Cloud Library v1.8 
+* OpenCV v3.2
+
+### Build
+
+```bash
+git clone https://github.com/karnikram/autocalib-sensor-extrinsics
+cd autocalib-sensor-extrinsics
+mkdir build && cd build
+cmake ..
+make
+./gui/autocalib-sensor-extrinsics
+```
+
+### Unit Tests
+
+TODO
+
+## Usage
+
+[![Demo Video](https://img.youtube.com/vi/IWMdQcNshFI/0.jpg)](https://www.youtube.com/watch?v=IWMdQcNshFI)
+
+
+### Source Reference
+
+The source files have been documented using doxygen style comment blocks. Running the command `doxygen doc/Doxyfile` from the root of the project directory will generate the documentation in HTML that can be browsed.
+
+
+### License
+
+The source code is released under the [new BSD license.](LICENSE)
+
+autocalib-sensor-extrinsics has been tested with Qt 5.9.5, OpenCV 3.2.0, and PCL 1.8 on Ubuntu 18.04. This contains research code, expect that it changes often and any fitness for a particular purpose is disclaimed.
+
+### References
+
+1) E. Fernandez-Moral, J. Gonzalez-Jimenez, P. Rives and V. Arevalo, "Extrinsic calibration of a set of range cameras in 5 seconds without pattern", IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS), Chicago (USA), 2014. [[link]](https://hal.inria.fr/hal-01010219)
+2) A. Perez-Yus, E. Fernandez-Moral, G. Lopez-Nicolas, JJ. Guerrero and P. Rives, "Extrinsic calibration of multiple RGB-D cameras from line observations", IEEE Robotics and Automation Letters 3 (1), 273-280. DOI: 10.1109/LRA.2017.2739104 [[link]](https://hal.inria.fr/hal-01581523)
+
+Please cite the above papers if this application was useful for your research.
+
+---
+
+This project is being developed as a part of [Google Summer of Code](https://summerofcode.withgoogle.com/projects/#4592205176504320).
+
+**Organization** : [Mobile Robot Programming Toolkit](https://github.com/mrpt/mrpt)<br>
+**Mentors** : [Eduardo Fernandez-Moral](https://github.com/EduFdez), [Jose Luis Blanco Claraco](https://github.com/jlblancoc), [Hunter Laux](https://github.com/jolting)
+
+**Weekly Progress Logs**: <http://karnikram.info/blog/gsoc-2018-progress/> <br>
+**Project discussion thread**: <https://github.com/mrpt/GSoC2018-discussions/issues/2>
+
+**Other minor contributions**: [PR #789](https://github.com/MRPT/mrpt/pull/789), [Issue #2364](https://github.com/PointCloudLibrary/pcl/issues/2364)
+
+## Roadmap
+- [x] Design wireframe for gui and setup a base gui skeleton
+- [x] Load the observations in the dataset into a treeview
+- [x] Setup dock widgets for the app setup, algorithm configuration
+- [x] Setup viewer window using qvtk and mrpt cqtglcanvas widgets
+- [x] Visualize the cloud and image of each observation in the viewers
+- [x] Synchronize the observations in the dataset to form corresponding sets
+- [x] Display the synchronized observations in a separate treeview along with reconfiguration options
+- [x] Visualize each observation set in the viewer using initial values
+- [x] Setup configuration widget for calibration from planes
+- [x] Extract planes from each observation in a set
+- [x] Visualize extracted planes and their properties in the viewer
+- [x] Perform data association between the segmented planes in each set
+- [x] Display the associated planes together in the main viewer with initial calibration values
+- [x] Setup configuration widget for calibration from lines
+- [x] Create and load config file for all the app parameters
+- [x] Extract 2D lines from images and visualize
+- [x] Extract 3D lines from range images and visualize
+- [x] Perform data association between the extracted 3D lines in each set
+- [x] Display the associated 3D lines together in the main viewer
+- [x] Integrate rotation solvers
+- [x] Integrate translation solvers
+- [x] Setup widget for editing initial calibration values loaded from config file 
+- [x] Add option to switch between sensors in the viewer for multi sensor calibration
+- [ ] Unit testing for solvers
+- [ ] Integrate ransac
+- [ ] Visualize final result
+
+<br><br>
+
+<p align="middle">
+	<img src="doc/images/mrpt-logo.png" width="300"/>
+	<img src="doc/images/gsoc-logo.png" width="400"/>
+</p>
+

cmake_modules/script_MRPT.cmake

@@ -0,0 +1,35 @@
+
+SET(RV_HAS_MRPT 0)
+#SET(RV_HAS_MRPT_BASE 0)
+
+# Leave at the user's choice to disable this library:
+OPTION(DISABLE_MRPT "Forces NOT using MRPT, even if it is found by CMake" "OFF")
+MARK_AS_ADVANCED(DISABLE_MRPT)
+
+IF(NOT DISABLE_MRPT)
+	# MRPT library:
+	# --------------------------------------------
+	#find_package(MRPT COMPONENTS base hwdrivers obs slam maps graphslam pbmap QUIET)
+	find_package(MRPT COMPONENTS obs serialization rtti maps gui pbmap QUIET)
+	IF(MRPT_FOUND)
+
+		SET(RV_HAS_MRPT 1)
+		SET(RV_HAS_MRPT_BASE 1)
+		add_definitions(-DRV_HAS_MRPT=1)
+		#add_definitions(-D_HAS_MRPT_BASE=${RV_HAS_MRPT_BASE})
+
+		INCLUDE_DIRECTORIES(${MRPT_INCLUDE_DIRS})
+		link_directories(${MRPT_LIBRARY_DIRS})
+		add_definitions(${MRPT_DEFINITIONS})
+
+		IF($ENV{VERBOSE})
+			MESSAGE(STATUS "MRPT:")
+			MESSAGE(STATUS " Include dirs: ${MRPT_INCLUDE_DIRS}")
+			MESSAGE(STATUS " Library dirs: ${MRPT_LIBRARY_DIRS}")
+			MESSAGE(STATUS " Definitions : ${MRPT_DEFINITIONS}")
+			MESSAGE(STATUS " Libraries   : ${MRPT_LIBRARIES}")
+		ENDIF($ENV{VERBOSE})
+
+	ENDIF(MRPT_FOUND)
+
+ENDIF(NOT DISABLE_MRPT)

config_files/app_config_checkerboard.ini

@@ -0,0 +1,54 @@
+# Master configuration file for all the funcitonality within the app.
+
+[rawlog]
+path=/home/karnik/dataset/checkerboard.rawlog
+
+[initial_calibration]
+#transformation matrices for the sensors in the rawlog
+
+RGBD_1=[1 0 0 0; 0 1 0 0; 0 0 1 0; 0 0 0 1]
+RGBD_2=[0.7071 0 0.7071 0; 0 1 0 0; -0.7071 0 0.7071 0; 0 0 0 1]
+
+[initial_uncertainty]
+#The angular and distance uncertainty of each sensor in the rawlog
+#Units are degrees and meters
+
+RGBD_1=[10 0.1]
+RGBD_2=[10 0.1]
+
+[grouping_observations]
+#maximum acceptable delay between observations in milliseconds
+max_delay=30
+
+[grouped_observations]
+#determines the number of grouped observation sets to use for calibration
+downsample_factor=1
+
+[plane_segmentation]
+#params for pcl integral normal estimation method
+normal_estimation_method=COVARIANCE_MATRIX
+depth_dependent_smoothing=true
+max_depth_change_factor=0.02
+normal_smoothing_size=10.00
+
+#params for pcl organized multiplane segmentation
+angle_threshold=4.00
+min_inliers_frac=0.001
+distance_threshold=0.05
+max_curvature=0.1
+
+#params for removing duplicate planes
+max_cos_normal=0.998
+dist_centre_plane_threshold=0.1 //min distance between center points of planes
+proximity_threshold=0.4 //min distance between the closest points of planes
+
+[line_segmentation]
+canny_low_threshold=150
+canny_high_to_low_ratio=3
+canny_kernel_size=3
+hough_threshold=50
+
+[solver]
+max_iters=10
+min_update=0.00001
+convergence_error=0.00001

config_files/app_config_livingroom.ini

@@ -0,0 +1,60 @@
+# Master configuration file for all the funcitonality within the app.
+
+[rawlog]
+path=/home/karnik/dataset/livingroom.rawlog
+
+[initial_calibration]
+#transformation matrices for the sensors in the rawlog
+
+RGBD_1=[0.70711 0 0.70711 0.27100; -0.70711 0 0.70711 -0.03500; 0 -1 0 1.0150; 0 0 0 1]
+RGBD_2=[0 0 1 0.2400; -1 0 0 -0.0490; 0 -1 0 1.0150; 0 0 0 1]
+RGBD_3=[0.70711 0 -0.70711 0.27100; 0.70711 0 0.70711 0.02700; 0 -1 0 1.0150; 0 0 0 1]
+RGBD_4=[1 0 0 0.28500; 0 0 1 -0.0400; 0 -1 0 1.0150; 0 0 0 1]
+
+[initial_uncertainty]
+
+#The initial angular and distance uncertainty of each sensor in the rawlog
+#Units are degrees and meters
+
+RGBD_1=[10 0.1]
+RGBD_2=[10 0.1]
+RGBD_3=[10 0.1]
+RGBD_4=[10 0.1]
+
+[grouping_observations]
+#maximum acceptable delay between observations in milliseconds
+max_delay=30
+
+[grouped_observations]
+#determines the number of grouped observation sets to use for calibration
+downsample_factor=1
+
+[plane_segmentation]
+#params for pcl integral normal estimation method
+normal_estimation_method=COVARIANCE_MATRIX
+depth_dependent_smoothing=true
+max_depth_change_factor=0.02
+normal_smoothing_size=10.00
+
+#params for pcl organized multiplane segmentation
+angle_threshold=4.00
+min_inliers_frac=0.001
+distance_threshold=0.05
+max_curvature=0.1
+
+#params for removing duplicate planes
+max_cos_normal=0.998
+dist_centre_plane_threshold=0.1 //min distance between center points of planes
+proximity_threshold=0.4 //min distance between the closest points of planes
+
+[line_segmentation]
+canny_low_threshold=150
+canny_high_to_low_ratio=3
+canny_kernel_size=3
+hough_threshold=50
+
+[solver]
+max_iters=10
+min_update=0.00001
+convergence_error=0.00001
+