From 551235b28c7a1ab4c0b0b4002fc3824e0e5209c4 Mon Sep 17 00:00:00 2001 From: Georges Berenger Date: Wed, 1 Feb 2023 15:06:23 -0800 Subject: [PATCH] New vrstool option: --decode Summary: The new --decode is a check that try to decode every image. This validates RecordFormat and DataLayout definitions, and image decoding to a PixelFrame. A side benefit is we can test raw decoding speed, not just raw speed. By design, it's single threaded. Reviewed By: kiminoue7 Differential Revision: D42399615 fbshipit-source-id: c95cdc3b5f2f64226698ef24c2e0aec448b18c6f --- tools/vrs/VrsCommand.cpp | 7 ++ tools/vrs/VrsCommand.h | 1 + vrs/utils/Validation.cpp | 168 ++++++++++++++++++++++++++++++++++++--- vrs/utils/Validation.h | 4 +- 4 files changed, 167 insertions(+), 13 deletions(-) diff --git a/tools/vrs/VrsCommand.cpp b/tools/vrs/VrsCommand.cpp index 3081f496..30f99ecc 100644 --- a/tools/vrs/VrsCommand.cpp +++ b/tools/vrs/VrsCommand.cpp @@ -63,6 +63,7 @@ const char* sCommands[] = { "checksums", "checksum-verbatim", "hexdump", + "decode", "compare", "compare-verbatim", "debug", @@ -110,6 +111,7 @@ const CommandSpec& getCommandSpec(Command cmd) { {Command::Checksums, 1}, {Command::ChecksumVerbatim, 1, Details::None, false}, {Command::Hexdump, 1}, + {Command::Decode, 1, Details::MainCounters}, {Command::Compare, 1000, Details::MainCounters}, {Command::CompareVerbatim, 1000, Details::None, false}, {Command::Debug, 1, Details::None, false}, @@ -183,6 +185,8 @@ void printHelp(const string& appName) { << endl << CMD("Check that a file can be read (checks integrity)", "check [filter-options]") + << CMD("Check that a file can be decoded (record-format integrity and image decompression)", + "decode [filter-options]") << CMD("Calculate a checksum for the whole file, at the VRS data level", "checksum [filter-options]") << CMD("Calculate checksums for each part of the VRS file", @@ -453,6 +457,9 @@ int VrsCommand::runCommands() { case Command::Check: cout << checkRecords(filteredReader, copyOptions, CheckType::Check) << endl; break; + case Command::Decode: + cout << checkRecords(filteredReader, copyOptions, CheckType::Decode) << endl; + break; case Command::Checksum: cout << checkRecords(filteredReader, copyOptions, CheckType::Checksum) << endl; break; diff --git a/tools/vrs/VrsCommand.h b/tools/vrs/VrsCommand.h index fc4e4d53..27da9d0a 100644 --- a/tools/vrs/VrsCommand.h +++ b/tools/vrs/VrsCommand.h @@ -37,6 +37,7 @@ enum class Command { Checksums, ChecksumVerbatim, Hexdump, + Decode, Compare, CompareVerbatim, Debug, diff --git a/vrs/utils/Validation.cpp b/vrs/utils/Validation.cpp index 1fc6615b..05c5cda2 100644 --- a/vrs/utils/Validation.cpp +++ b/vrs/utils/Validation.cpp @@ -28,7 +28,11 @@ #include #include +#include +#include #include +#include +#include #include using namespace std; @@ -123,6 +127,7 @@ class RecordChecker : public StreamPlayer { } } break; case CheckType::ChecksumVerbatim: // not handled here + case CheckType::Decode: // not handled here case CheckType::None: case CheckType::Check: case CheckType::COUNT: @@ -156,13 +161,16 @@ bool iterateChecker( FilteredFileReader& reader, size_t& outDecodedRecords, bool& outNoError, - ThrottledWriter* throttledWriter) { + ThrottledWriter* throttledWriter, + double& outDuration) { outDecodedRecords = 0; outNoError = true; if (!reader.timeRangeValid()) { cerr << "Time Range invalid: " << reader.getTimeConstraintDescription() << endl; + outDuration = 0; return false; } + double beforeTime = os::getTimestampSec(); reader.iterateAdvanced( [&outDecodedRecords, &outNoError]( RecordFileReader& recordFileReader, const IndexRecord::RecordInfo& record) { @@ -174,6 +182,7 @@ bool iterateChecker( for (auto id : reader.filter.streams) { reader.reader.setStreamPlayer(id, nullptr); } + outDuration = os::getTimestampSec() - beforeTime; return true; } @@ -208,6 +217,134 @@ string mapAsJson(const map& strMap) { return jDocumentToJsonString(document); } +class DecodeChecker : public VideoRecordFormatStreamPlayer { + public: + DecodeChecker(uint32_t& errorCount, uint32_t& imageCount) + : errorCount_{errorCount}, imageCount_{imageCount} {} + bool processRecordHeader(const CurrentRecord& record, DataReference& outDataReference) override { + if (VideoRecordFormatStreamPlayer::processRecordHeader(record, outDataReference)) { + return true; + } + if (record.recordSize > 0) { + errorCount_++; + } + return false; + } + void processRecord(const CurrentRecord& record, uint32_t readSize) override { + processSuccess = true; + RecordFormatStreamPlayer::processRecord(record, readSize); + size_t unreadBytes = record.reader->getUnreadBytes(); + if (unreadBytes > 0) { + processSuccess = false; + XR_LOGW( + "{} - {}: {} bytes unread out of {} bytes.", + record.streamId.getNumericName(), + Record::typeName(record.recordType), + unreadBytes, + record.recordSize); + } else if (!processSuccess) { + XR_LOGW( + "{} - {}: could not be decoded.", + record.streamId.getNumericName(), + Record::typeName(record.recordType)); + } + if (!processSuccess) { + errorCount_++; + } + } + bool onDataLayoutRead(const CurrentRecord& r, size_t /* blockIndex */, DataLayout& dl) override { + return true; + } + bool onImageRead(const CurrentRecord& record, size_t blockIndex, const ContentBlock& cb) + override { + if (cb.image().getImageFormat() == ImageFormat::VIDEO) { + PixelFrame frame; + return isSuccess(tryToDecodeFrame(frame, record, cb) == 0, cb.getContentType()); + } else { + shared_ptr frame; + return isSuccess(PixelFrame::readFrame(frame, record.reader, cb), cb.getContentType()); + } + return onUnsupportedBlock(record, blockIndex, cb); + } + bool onAudioRead(const CurrentRecord& record, size_t blockIndex, const ContentBlock& cb) + override { + return onUnsupportedBlock(record, blockIndex, cb); + } + bool onCustomBlockRead(const CurrentRecord& rec, size_t blkIdx, const ContentBlock& cb) override { + return onUnsupportedBlock(rec, blkIdx, cb); + } + bool onUnsupportedBlock(const CurrentRecord& rec, size_t blkIdx, const ContentBlock& cb) + override { + size_t blockSize = cb.getBlockSize(); + if (blockSize == ContentBlock::kSizeUnknown) { + XR_LOGW("Block size for {} unknown.", cb.asString()); + return isSuccess(false); + } + vector data(blockSize); + return isSuccess(rec.reader->read(data) == 0); + } + bool isSuccess(bool success, ContentType contentType = ContentType::EMPTY) { + if (!success) { + processSuccess = false; + } else if (contentType == ContentType::IMAGE) { + imageCount_++; + } + return success; + } + + private: + uint32_t& errorCount_; + uint32_t& imageCount_; + bool processSuccess{false}; +}; + +string decodeValidation(FilteredFileReader& filteredReader, const CopyOptions& copyOptions) { + uint32_t decodeErrorCount{0}; + uint32_t imageCount{0}; + vector> checkers; + for (auto id : filteredReader.filter.streams) { + checkers.emplace_back(make_unique(decodeErrorCount, imageCount)); + filteredReader.reader.setStreamPlayer(id, checkers.back().get()); + } + + double startTimestamp, endTimestamp; + filteredReader.getConstrainedTimeRange(startTimestamp, endTimestamp); + filteredReader.preRollConfigAndState(); // make sure to copy most recent config & state records + + ThrottledWriter throttledWriter(copyOptions); + throttledWriter.initTimeRange(startTimestamp, endTimestamp); + + size_t readRecordCount; + bool noError; + double timeSpent = 0; + if (!iterateChecker(filteredReader, readRecordCount, noError, &throttledWriter, timeSpent)) { + return ""; + } + throttledWriter.closeFile(); + + size_t decodedCount = + readRecordCount >= decodeErrorCount ? readRecordCount - decodeErrorCount : 0; + double successRate = decodedCount * 100. / readRecordCount; + if (copyOptions.jsonOutput) { + double duration = endTimestamp - startTimestamp; + double mbPerSec = + duration > 0 ? filteredReader.reader.getTotalSourceSize() / (duration * 1024 * 1024) : -1; + return asJson(noError, readRecordCount, duration, mbPerSec, decodedCount, successRate); + } + if (noError && decodeErrorCount == 0) { + return fmt::format( + "Decoded {} records, {} images, in {}, no errors.", + decodedCount, + imageCount, + helpers::humanReadableDuration(timeSpent)); + } + return fmt::format( + "Failure! Decoded {} records out of {}, {:.2f}% good.", + decodedCount, + readRecordCount, + successRate); +} + } // namespace namespace vrs::utils { @@ -217,7 +354,10 @@ string checkRecords( const CopyOptions& copyOptions, CheckType checkType) { if (!filteredReader.reader.isOpened()) { - return ""; + return {}; + } + if (checkType == CheckType::Decode) { + return decodeValidation(filteredReader, copyOptions); } vector> checkers; // Instance ids are assigned by VRS and can not be relied upon, though ordering is @@ -238,7 +378,8 @@ string checkRecords( size_t decodedCount; bool noError; - if (!iterateChecker(filteredReader, decodedCount, noError, &throttledWriter)) { + double timeSpent = 0; + if (!iterateChecker(filteredReader, decodedCount, noError, &throttledWriter, timeSpent)) { return ""; } throttledWriter.closeFile(); @@ -293,6 +434,7 @@ string checkRecords( } break; case CheckType::ChecksumVerbatim: case CheckType::HexDump: + case CheckType::Decode: case CheckType::None: case CheckType::Check: case CheckType::COUNT: @@ -316,14 +458,17 @@ string checkRecords( duration > 0 ? filteredReader.reader.getTotalSourceSize() / (duration * 1024 * 1024) : -1; return asJson(noError, recordCount, duration, mbPerSec, decodedCount, successRate); } - stringstream ss; if (noError) { - ss << "Decoded " << decodedCount << " records, no errors."; - } else { - ss << "Failure! Decoded " << decodedCount << " records out of " << recordCount << ", " << fixed - << setprecision(2) << successRate << "% good."; - } - return ss.str(); + return fmt::format( + "Checked {} records in {}, no errors.", + decodedCount, + helpers::humanReadableDuration(timeSpent)); + } + return fmt::format( + "Failure! Checked {} records out of {}, {:.2f}% good.", + decodedCount, + recordCount, + successRate); } string recordsChecksum(const string& path, bool showProgress) { @@ -643,7 +788,8 @@ bool compareVRSfiles( first.reader.setStreamPlayer(iter.first, checkers.back().get()); } size_t decodedCount; - if (!iterateChecker(first, decodedCount, noError, &throttledWriter)) { + double timeSpent = 0; + if (!iterateChecker(first, decodedCount, noError, &throttledWriter, timeSpent)) { return false; } if (!noError) { diff --git a/vrs/utils/Validation.h b/vrs/utils/Validation.h index 1b539183..847b5dea 100644 --- a/vrs/utils/Validation.h +++ b/vrs/utils/Validation.h @@ -25,13 +25,13 @@ namespace vrs::utils { -enum class CheckType { None, Check, Checksum, ChecksumVerbatim, Checksums, HexDump, COUNT }; +enum class CheckType { None, Check, Checksum, ChecksumVerbatim, Checksums, HexDump, Decode, COUNT }; /// Check a VRS file by reading all its records & counting errors. /// The file should be open & filters applied already. /// @param filteredReader: source file with record filtering /// @param copyOptions: to disable progress logging. Other parameters are not applicable. -/// @param checkType: which type of check is expected. +/// @param checkType: which type of check is expected, except ChecksumVerbatim (see API below). std::string checkRecords( FilteredFileReader& filteredReader, const CopyOptions& copyOptions,