diff --git a/src/aliceVision/keyframe/KeyframeSelector.cpp b/src/aliceVision/keyframe/KeyframeSelector.cpp index 2f0e462a71..2d9f6b133c 100644 --- a/src/aliceVision/keyframe/KeyframeSelector.cpp +++ b/src/aliceVision/keyframe/KeyframeSelector.cpp @@ -16,6 +16,7 @@ #include #include #include +#include namespace fs = boost::filesystem; @@ -320,24 +321,31 @@ bool KeyframeSelector::computeScores(const std::size_t rescaledWidthSharpness, c _frameWidth = 0; _frameHeight = 0; - // Create feeds and count minimum number of frames + // Create single feed and count minimum number of frames std::size_t nbFrames = std::numeric_limits::max(); - std::vector> feeds; for (std::size_t mediaIndex = 0; mediaIndex < _mediaPaths.size(); ++mediaIndex) { const auto& path = _mediaPaths.at(mediaIndex); // Create a feed provider per mediaPaths - feeds.push_back(std::make_unique(path)); - const auto& feed = *feeds.back(); + auto feed = std::make_unique(path); // Check if feed is initialized - if (!feed.isInit()) { + if (!feed->isInit()) { ALICEVISION_THROW(std::invalid_argument, "Cannot initialize the FeedProvider with " << path); } - // Update minimum number of frames - nbFrames = std::min(nbFrames, (size_t)feed.nbFrames()); + // Number of frames in the rig might slightly differ + nbFrames = std::min(nbFrames, static_cast(feed->nbFrames())); + + if (mediaIndex == 0) { + // Read first image and set _frameWidth and _frameHeight, since the feeds have been initialized + feed->goToFrame(0); + cv::Mat mat = readImage(*feed, rescaledWidthFlow); + // Will be used later on to determine the motion accumulation step + _frameWidth = mat.size().width; + _frameHeight = mat.size().height; + } } // Check if minimum number of frame is zero @@ -345,6 +353,68 @@ bool KeyframeSelector::computeScores(const std::size_t rescaledWidthSharpness, c ALICEVISION_THROW(std::invalid_argument, "One or multiple medias can't be found or is empty!"); } + // With the number of threads available and the number of frames to process known, + // blocks can be prepared for multi-threading + int nbThreads = omp_get_max_threads(); + + std::size_t blockSize = (nbFrames / static_cast(nbThreads)) + 1; + + // If a block contains less than _minBlockSize frames (when there are lots of available threads for a small number + // of frames, for example), resize it: less threads will be spawned, but since new FeedProvider objects need to be + // created for each thread, we prevent spawing thread that will need to create FeedProvider objects + // for very few frames. + if (blockSize < _minBlockSize && nbFrames >= _minBlockSize) { + blockSize = _minBlockSize; + nbThreads = static_cast(nbFrames / blockSize) + 1; // +1 to ensure that every frame in processed by a thread + } + + std::vector threads; + ALICEVISION_LOG_INFO("Splitting " << nbFrames << " frames into " << nbThreads << " threads of size " << blockSize << "."); + + for (std::size_t i = 0; i < nbThreads; i++) { + std::size_t startFrame = static_cast(std::max(0, static_cast(i * blockSize) - 1)); + std::size_t endFrame = std::min(i * blockSize + blockSize, nbFrames); + + // If there is an extra thread with no new frames to process, skip it. + // This might occur as a consequence of the "+1" when adjusting the number of threads. + if (startFrame >= nbFrames) { + break; + } + + ALICEVISION_LOG_DEBUG("Starting thread to compute scores for frame " << startFrame << " to " << endFrame << "."); + + threads.push_back(std::thread(&KeyframeSelector::computeScoresProc, this, startFrame, endFrame, nbFrames, + rescaledWidthSharpness, rescaledWidthFlow, sharpnessWindowSize, flowCellSize, + skipSharpnessComputation)); + } + + for (auto &th : threads) { + th.join(); + } + + return true; +} + +bool KeyframeSelector::computeScoresProc(const std::size_t startFrame, const std::size_t endFrame, + const std::size_t nbFrames, const std::size_t rescaledWidthSharpness, + const std::size_t rescaledWidthFlow, const std::size_t sharpnessWindowSize, + const std::size_t flowCellSize, const bool skipSharpnessComputation) +{ + std::vector> feeds; + + for (std::size_t mediaIndex = 0; mediaIndex < _mediaPaths.size(); ++mediaIndex) { + const auto& path = _mediaPaths.at(mediaIndex); + + // Create a feed provider per mediaPaths + feeds.push_back(std::make_unique(path)); + const auto& feed = *feeds.back(); + + // Check if feed is initialized + if (!feed.isInit()) { + ALICEVISION_THROW(std::invalid_argument, "Cannot initialize the FeedProvider with " << path); + } + } + // Feed provider variables image::Image image; // original image camera::PinholeRadialK3 queryIntrinsics; // image associated camera intrinsics @@ -354,26 +424,26 @@ bool KeyframeSelector::computeScores(const std::size_t rescaledWidthSharpness, c // Feed and metadata initialization for (std::size_t mediaIndex = 0; mediaIndex < feeds.size(); ++mediaIndex) { // First frame with offset - feeds.at(mediaIndex)->goToFrame(0); + feeds.at(mediaIndex)->goToFrame(startFrame); if (!feeds.at(mediaIndex)->readImage(image, queryIntrinsics, currentImgName, hasIntrinsics)) { ALICEVISION_THROW(std::invalid_argument, "Cannot read media first frame " << _mediaPaths[mediaIndex]); } } - std::size_t currentFrame = 0; + std::size_t currentFrame = startFrame; cv::Mat currentMatSharpness; // OpenCV matrix for the sharpness computation cv::Mat previousMatFlow, currentMatFlow; // OpenCV matrices for the optical flow computation auto ptrFlow = cv::optflow::createOptFlow_DeepFlow(); - while (currentFrame < nbFrames) { + while (currentFrame < endFrame) { double minimalSharpness = skipSharpnessComputation ? 1.0f : std::numeric_limits::max(); double minimalFlow = std::numeric_limits::max(); for (std::size_t mediaIndex = 0; mediaIndex < feeds.size(); ++mediaIndex) { auto& feed = *feeds.at(mediaIndex); - if (currentFrame > 0) { // Get currentFrame - 1 for the optical flow computation + if (currentFrame > startFrame) { // Get currentFrame - 1 for the optical flow computation previousMatFlow = readImage(feed, rescaledWidthFlow); feed.goToNextFrame(); } @@ -397,14 +467,18 @@ bool KeyframeSelector::computeScores(const std::size_t rescaledWidthSharpness, c // currentFrame + 2 = next frame to evaluate with indexing starting at 1, for display reasons ALICEVISION_LOG_WARNING("Invalid or missing frame " << currentFrame + 1 << ", attempting to read frame " << currentFrame + 2 << "."); + + { + // Push dummy scores for the frame that was skipped + const std::scoped_lock lock(_mutex); + _sharpnessScores[currentFrame] = -1.f; + _flowScores[currentFrame] = -1.f; + } + success = feed.goToFrame(++currentFrame); if (success) { currentMatSharpness = readImage(feed, rescaledWidthSharpness); } - - // Push dummy scores for the frame that was skipped - _sharpnessScores.push_back(-1.f); - _flowScores.push_back(-1.f); } } } @@ -415,11 +489,6 @@ bool KeyframeSelector::computeScores(const std::size_t rescaledWidthSharpness, c currentMatFlow = readImage(feed, rescaledWidthFlow); } - if (_frameWidth == 0 && _frameHeight == 0) { // Will be used later on to determine the motion accumulation step - _frameWidth = currentMatFlow.size().width; - _frameHeight = currentMatFlow.size().height; - } - // Compute sharpness if (!skipSharpnessComputation) { const double sharpness = computeSharpness(currentMatSharpness, sharpnessWindowSize); @@ -427,20 +496,23 @@ bool KeyframeSelector::computeScores(const std::size_t rescaledWidthSharpness, c } // Compute optical flow - if (currentFrame > 0) { + if (currentFrame > startFrame) { const double flow = estimateFlow(ptrFlow, currentMatFlow, previousMatFlow, flowCellSize); minimalFlow = std::min(minimalFlow, flow); } - ALICEVISION_LOG_INFO("Finished processing frame " << currentFrame + 1 << "/" << nbFrames); + std::string rigInfo = feeds.size() > 1 ? " (media " + std::to_string(mediaIndex + 1) + "/" + std::to_string(feeds.size()) + ")" : ""; + ALICEVISION_LOG_INFO("Finished processing frame " << currentFrame + 1 << "/" << nbFrames << rigInfo); } - // Save scores for the current frame - _sharpnessScores.push_back(minimalSharpness); - _flowScores.push_back(currentFrame > 0 ? minimalFlow : -1.f); + { + // Save scores for the current frame + const std::scoped_lock lock(_mutex); + _sharpnessScores[currentFrame] = minimalSharpness; + _flowScores[currentFrame] = currentFrame > startFrame ? minimalFlow : -1.f; + } ++currentFrame; } - return true; } @@ -508,8 +580,7 @@ bool KeyframeSelector::writeSelection(const std::vector& brands, metadata.push_back(oiio::ParamValue("Exif:FocalLength", mmFocals[id])); metadata.push_back(oiio::ParamValue("Exif:ImageUniqueID", std::to_string(getRandomInt()))); metadata.push_back(oiio::ParamValue("Orientation", orientation)); // Will not propagate for PNG outputs - if (outputExtension != "jpg") // TODO: propagate pixelAspectRatio properly for JPG - metadata.push_back(oiio::ParamValue("PixelAspectRatio", pixelAspectRatio)); + metadata.push_back(oiio::ParamValue("PixelAspectRatio", pixelAspectRatio)); fs::path folder = _outputFolder; std::ostringstream filenameSS; @@ -757,33 +828,52 @@ double KeyframeSelector::computeSharpness(const cv::Mat& grayscaleImage, const s cv::Laplacian(grayscaleImage, laplacian, CV_64F); cv::integral(laplacian, sum, squaredSum); - double totalCount = windowSize * windowSize; double maxstd = 0.0; + int x, y; - // TODO: do not slide the window pixel by pixel to speed up computations // Starts at 1 because the integral image is padded with 0s on the top and left borders - for (int y = 1; y < sum.rows - windowSize; ++y) { - for (int x = 1; x < sum.cols - windowSize; ++x) { - double tl = sum.at(y, x); - double tr = sum.at(y, x + windowSize); - double bl = sum.at(y + windowSize, x); - double br = sum.at(y + windowSize, x + windowSize); - const double s1 = br + tl - tr - bl; - - tl = squaredSum.at(y, x); - tr = squaredSum.at(y, x + windowSize); - bl = squaredSum.at(y + windowSize, x); - br = squaredSum.at(y + windowSize, x + windowSize); - const double s2 = br + tl - tr - bl; - - const double std2 = std::sqrt((s2 - (s1 * s1) / totalCount) / totalCount); - maxstd = std::max(maxstd, std2); + for (y = 1; y < sum.rows - windowSize; y += windowSize / 4) { + for (x = 1; x < sum.cols - windowSize; x += windowSize / 4) { + maxstd = std::max(maxstd, computeSharpnessStd(sum, squaredSum, x, y, windowSize)); } + + // Compute sharpness over the last part of the image for windowSize along the x-axis; + // the overlap with the previous window might be greater than the previous ones + if (x >= sum.cols - windowSize) { + x = sum.cols - windowSize - 1; + maxstd = std::max(maxstd, computeSharpnessStd(sum, squaredSum, x, y, windowSize)); + } + } + + // Compute sharpness over the last part of the image for windowSize along the y-axis; + // the overlap with the previous window might be greater than the previous ones + if (y >= sum.rows - windowSize) { + y = sum.rows - windowSize - 1; + maxstd = std::max(maxstd, computeSharpnessStd(sum, squaredSum, x, y, windowSize)); } return maxstd; } +const double KeyframeSelector::computeSharpnessStd(const cv::Mat& sum, const cv::Mat& squaredSum, const int x, + const int y, const int windowSize) +{ + const double totalCount = windowSize * windowSize; + double tl = sum.at(y, x); + double tr = sum.at(y, x + windowSize); + double bl = sum.at(y + windowSize, x); + double br = sum.at(y + windowSize, x + windowSize); + const double s1 = br + tl - tr - bl; + + tl = squaredSum.at(y, x); + tr = squaredSum.at(y, x + windowSize); + bl = squaredSum.at(y + windowSize, x); + br = squaredSum.at(y + windowSize, x + windowSize); + const double s2 = br + tl - tr - bl; + + return std::sqrt((s2 - (s1 * s1) / totalCount) / totalCount); +} + double KeyframeSelector::estimateFlow(const cv::Ptr& ptrFlow, const cv::Mat& grayscaleImage, const cv::Mat& previousGrayscaleImage, const std::size_t cellSize) { diff --git a/src/aliceVision/keyframe/KeyframeSelector.hpp b/src/aliceVision/keyframe/KeyframeSelector.hpp index 5ffdc59722..caaef3d03a 100644 --- a/src/aliceVision/keyframe/KeyframeSelector.hpp +++ b/src/aliceVision/keyframe/KeyframeSelector.hpp @@ -17,8 +17,8 @@ #include #include -#include #include +#include #include #include #include @@ -181,6 +181,15 @@ class KeyframeSelector _maxOutFrames = nbFrames; } + /** + * @brief Set the minimum size of the blocks of frames for the multi-threading + * @param[in] blockSize minimum number of frames in a block for a thread to be spawned + */ + void setMinBlockSize(std::size_t blockSize) + { + _minBlockSize = blockSize; + } + /** * @brief Get the minimum frame step parameter for the processing algorithm * @return minimum number of frames between two keyframes @@ -219,7 +228,7 @@ class KeyframeSelector private: /** - * @brief Read an image from a feed provider into a grayscale OpenCV matrix, and rescale it if a size is provided. + * @brief Read an image from a feed provider into a grayscale OpenCV matrix, and rescale it if a size is provided * @param[in] feed The feed provider * @param[in] width The width to resize the input image to. The height will be adjusted with respect to the size ratio. * There will be no resizing if this parameter is set to 0 @@ -227,6 +236,28 @@ class KeyframeSelector */ cv::Mat readImage(dataio::FeedProvider &feed, std::size_t width = 0); + + /** + * @brief Compute the sharpness and optical flow scores for the input media paths for a given range of frames + * @param[in] startFrame the index of the first frame to compute the scores for + * @param[in] endFrame the index of the last frame to compute the scores for + * @param[in] nbFrames the total number of frames in the sequence + * @param[in] rescaledWidthSharpness the width to resize the input frames to before using them to compute the + * sharpness scores (if equal to 0, no rescale will be performed) + * @param[in] rescaledWidthFlow the width to resize the input frames to before using them to compute the + * motion scores (if equal to 0, no rescale will be performed) + * @param[in] sharpnessWindowSize the size of the sliding window used to compute sharpness scores, in pixels + * @param[in] flowCellSize the size of the cells within a frame that are used to compute the optical flow scores, + * in pixels + * @param[in] skipSharpnessComputation if true, the sharpness score computations will not be performed and a fixed + * sharpness score will be given to all the input frames + * @return true if the scores have been successfully computed for all frames, false otherwise + */ + bool computeScoresProc(const std::size_t startFrame, const std::size_t endFrame, const std::size_t nbFrames, + const std::size_t rescaledWidthSharpness, const std::size_t rescaledWidthFlow, + const std::size_t sharpnessWindowSize, const std::size_t flowCellSize, + const bool skipSharpnessComputation); + /** * @brief Compute the sharpness scores for an input grayscale frame with a sliding window * @param[in] grayscaleImage the input grayscale matrix of the frame @@ -235,6 +266,17 @@ class KeyframeSelector */ double computeSharpness(const cv::Mat& grayscaleImage, const std::size_t windowSize); + /** + * @brief Compute the standard deviation of the local averaged Laplacian in an image + * @param sum The integral image of the Laplacian of a given image + * @param squaredSum The squared integral image of the Laplacian of a given image + * @param x The x-coordinate of the top-left corner of the window for the local standard deviation computation + * @param y The y-coordinate of the top-left corner of the window for the local standard deviation computation + * @param windowSize The size of the window along the x- and y-axis for the local standard deviation computation + * @return a const double value representating the local standard deviation of the Laplacian + */ + const double computeSharpnessStd(const cv::Mat& sum, const cv::Mat& squaredSum, const int x, const int y, const int windowSize); + /** * @brief Estimate the optical flow score for an input grayscale frame based on its previous frame cell by cell * @param[in] ptrFlow the OpenCV's DenseOpticalFlow object @@ -333,10 +375,13 @@ class KeyframeSelector /// Minimum number of output frames unsigned int _minOutFrames = 10; + /// Minimum block size for multi-threading + std::size_t _minBlockSize = 10; + /// Sharpness scores for each frame - std::vector _sharpnessScores; + std::map _sharpnessScores; /// Optical flow scores for each frame - std::vector _flowScores; + std::map _flowScores; /// Vector containing 1s for frames that have been selected, 0 for those which have not std::vector _selectedFrames; @@ -357,7 +402,10 @@ class KeyframeSelector std::map> _keyframesPaths; /// Map score vectors with names for export - std::map*> scoresMap; + std::map*> scoresMap; + + /// Mutex to ensure thread-safe operations + mutable std::mutex _mutex; }; } // namespace keyframe diff --git a/src/software/utils/main_keyframeSelection.cpp b/src/software/utils/main_keyframeSelection.cpp index 0fb89f25fa..47e35999f6 100644 --- a/src/software/utils/main_keyframeSelection.cpp +++ b/src/software/utils/main_keyframeSelection.cpp @@ -19,7 +19,7 @@ // These constants define the current software version. // They must be updated when the command line is changed. #define ALICEVISION_SOFTWARE_VERSION_MAJOR 4 -#define ALICEVISION_SOFTWARE_VERSION_MINOR 0 +#define ALICEVISION_SOFTWARE_VERSION_MINOR 1 using namespace aliceVision; @@ -55,6 +55,7 @@ int aliceVision_main(int argc, char** argv) image::EStorageDataType exrDataType = // storage data type for EXR output files image::EStorageDataType::Float; bool renameKeyframes = false; // name selected keyframes as consecutive frames instead of using their index as a name + std::size_t minBlockSize = 10; // minimum number of frames in a block for multi-threading // Debug options bool exportScores = false; // export the sharpness and optical flow scores to a CSV file @@ -134,7 +135,9 @@ int aliceVision_main(int argc, char** argv) ("sharpnessWindowSize", po::value(&sharpnessWindowSize)->default_value(sharpnessWindowSize), "Size, in pixels, of the sliding window that is used to compute the sharpness score of a frame.") ("flowCellSize", po::value(&flowCellSize)->default_value(flowCellSize), - "Size, in pixels, of the cells within an input frame that are used to compute the optical flow scores."); + "Size, in pixels, of the cells within an input frame that are used to compute the optical flow scores.") + ("minBlockSize", po::value(&minBlockSize)->default_value(minBlockSize), + "Minimum number of frames processed by a single thread when multi-threading is used."); po::options_description debugParams("Debug parameters"); debugParams.add_options() @@ -221,6 +224,9 @@ int aliceVision_main(int argc, char** argv) } } + HardwareContext hwc = cmdline.getHardwareContext(); + omp_set_num_threads(hwc.getMaxThreads()); + // Initialize KeyframeSelector keyframe::KeyframeSelector selector(inputPaths, sensorDbPath, outputFolder, outputSfMDataKeyframes, outputSfMDataFrames); @@ -230,6 +236,7 @@ int aliceVision_main(int argc, char** argv) selector.setMaxFrameStep(maxFrameStep); selector.setMinOutFrames(minNbOutFrames); selector.setMaxOutFrames(maxNbOutFrames); + selector.setMinBlockSize(minBlockSize); if (flowVisualisationOnly) { bool exported = selector.exportFlowVisualisation(rescaledWidthFlow);