Skip to content

Commit

Permalink
New vrstool option: --decode
Browse files Browse the repository at this point in the history 
Summary:
The new --decode is a check that try to decode every image. This validates RecordFormat and DataLayout definitions, and image decoding to a PixelFrame.
A side benefit is we can test raw decoding speed, not just raw speed. By design, it's single threaded.

Reviewed By: kiminoue7

Differential Revision: D42399615

fbshipit-source-id: c95cdc3b5f2f64226698ef24c2e0aec448b18c6f
  • Loading branch information
@facebook-github-bot
Georges Berenger authored and facebook-github-bot committed Feb 1, 2023
1 parent 34ef8bb commit 551235b
Show file tree
Hide file tree
Showing 4 changed files with 167 additions and 13 deletions.
7 changes: 7 additions & 0 deletions tools/vrs/VrsCommand.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ const char* sCommands[] = {
"checksums",
"checksum-verbatim",
"hexdump",
"decode",
"compare",
"compare-verbatim",
"debug",
Expand Down Expand Up @@ -110,6 +111,7 @@ const CommandSpec& getCommandSpec(Command cmd) {
{Command::Checksums, 1},
{Command::ChecksumVerbatim, 1, Details::None, false},
{Command::Hexdump, 1},
{Command::Decode, 1, Details::MainCounters},
{Command::Compare, 1000, Details::MainCounters},
{Command::CompareVerbatim, 1000, Details::None, false},
{Command::Debug, 1, Details::None, false},
Expand Down Expand Up @@ -183,6 +185,8 @@ void printHelp(const string& appName) {
<< endl
<< CMD("Check that a file can be read (checks integrity)",
"check <file.vrs> [filter-options]")
<< CMD("Check that a file can be decoded (record-format integrity and image decompression)",
"decode <file.vrs> [filter-options]")
<< CMD("Calculate a checksum for the whole file, at the VRS data level",
"checksum <file.vrs> [filter-options]")
<< CMD("Calculate checksums for each part of the VRS file",
Expand Down Expand Up @@ -453,6 +457,9 @@ int VrsCommand::runCommands() {
case Command::Check:
cout << checkRecords(filteredReader, copyOptions, CheckType::Check) << endl;
break;
case Command::Decode:
cout << checkRecords(filteredReader, copyOptions, CheckType::Decode) << endl;
break;
case Command::Checksum:
cout << checkRecords(filteredReader, copyOptions, CheckType::Checksum) << endl;
break;
Expand Down
1 change: 1 addition & 0 deletions tools/vrs/VrsCommand.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ enum class Command {
Checksums,
ChecksumVerbatim,
Hexdump,
Decode,
Compare,
CompareVerbatim,
Debug,
Expand Down
168 changes: 157 additions & 11 deletions vrs/utils/Validation.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,11 @@

#include <vrs/FileHandlerFactory.h>
#include <vrs/helpers/Rapidjson.hpp>
#include <vrs/helpers/Strings.h>
#include <vrs/os/Time.h>
#include <vrs/utils/FilterCopy.h>
#include <vrs/utils/PixelFrame.h>
#include <vrs/utils/VideoRecordFormatStreamPlayer.h>
#include <vrs/utils/xxhash/xxhash.h>

using namespace std;
Expand Down Expand Up @@ -123,6 +127,7 @@ class RecordChecker : public StreamPlayer {
}
} break;
case CheckType::ChecksumVerbatim: // not handled here
case CheckType::Decode: // not handled here
case CheckType::None:
case CheckType::Check:
case CheckType::COUNT:
Expand Down Expand Up @@ -156,13 +161,16 @@ bool iterateChecker(
FilteredFileReader& reader,
size_t& outDecodedRecords,
bool& outNoError,
ThrottledWriter* throttledWriter) {
ThrottledWriter* throttledWriter,
double& outDuration) {
outDecodedRecords = 0;
outNoError = true;
if (!reader.timeRangeValid()) {
cerr << "Time Range invalid: " << reader.getTimeConstraintDescription() << endl;
outDuration = 0;
return false;
}
double beforeTime = os::getTimestampSec();
reader.iterateAdvanced(
[&outDecodedRecords, &outNoError](
RecordFileReader& recordFileReader, const IndexRecord::RecordInfo& record) {
Expand All @@ -174,6 +182,7 @@ bool iterateChecker(
for (auto id : reader.filter.streams) {
reader.reader.setStreamPlayer(id, nullptr);
}
outDuration = os::getTimestampSec() - beforeTime;
return true;
}

Expand Down Expand Up @@ -208,6 +217,134 @@ string mapAsJson(const map<string, string>& strMap) {
return jDocumentToJsonString(document);
}

class DecodeChecker : public VideoRecordFormatStreamPlayer {
public:
DecodeChecker(uint32_t& errorCount, uint32_t& imageCount)
: errorCount_{errorCount}, imageCount_{imageCount} {}
bool processRecordHeader(const CurrentRecord& record, DataReference& outDataReference) override {
if (VideoRecordFormatStreamPlayer::processRecordHeader(record, outDataReference)) {
return true;
}
if (record.recordSize > 0) {
errorCount_++;
}
return false;
}
void processRecord(const CurrentRecord& record, uint32_t readSize) override {
processSuccess = true;
RecordFormatStreamPlayer::processRecord(record, readSize);
size_t unreadBytes = record.reader->getUnreadBytes();
if (unreadBytes > 0) {
processSuccess = false;
XR_LOGW(
"{} - {}: {} bytes unread out of {} bytes.",
record.streamId.getNumericName(),
Record::typeName(record.recordType),
unreadBytes,
record.recordSize);
} else if (!processSuccess) {
XR_LOGW(
"{} - {}: could not be decoded.",
record.streamId.getNumericName(),
Record::typeName(record.recordType));
}
if (!processSuccess) {
errorCount_++;
}
}
bool onDataLayoutRead(const CurrentRecord& r, size_t /* blockIndex */, DataLayout& dl) override {
return true;
}
bool onImageRead(const CurrentRecord& record, size_t blockIndex, const ContentBlock& cb)
override {
if (cb.image().getImageFormat() == ImageFormat::VIDEO) {
PixelFrame frame;
return isSuccess(tryToDecodeFrame(frame, record, cb) == 0, cb.getContentType());
} else {
shared_ptr<PixelFrame> frame;
return isSuccess(PixelFrame::readFrame(frame, record.reader, cb), cb.getContentType());
}
return onUnsupportedBlock(record, blockIndex, cb);
}
bool onAudioRead(const CurrentRecord& record, size_t blockIndex, const ContentBlock& cb)
override {
return onUnsupportedBlock(record, blockIndex, cb);
}
bool onCustomBlockRead(const CurrentRecord& rec, size_t blkIdx, const ContentBlock& cb) override {
return onUnsupportedBlock(rec, blkIdx, cb);
}
bool onUnsupportedBlock(const CurrentRecord& rec, size_t blkIdx, const ContentBlock& cb)
override {
size_t blockSize = cb.getBlockSize();
if (blockSize == ContentBlock::kSizeUnknown) {
XR_LOGW("Block size for {} unknown.", cb.asString());
return isSuccess(false);
}
vector<char> data(blockSize);
return isSuccess(rec.reader->read(data) == 0);
}
bool isSuccess(bool success, ContentType contentType = ContentType::EMPTY) {
if (!success) {
processSuccess = false;
} else if (contentType == ContentType::IMAGE) {
imageCount_++;
}
return success;
}

private:
uint32_t& errorCount_;
uint32_t& imageCount_;
bool processSuccess{false};
};

string decodeValidation(FilteredFileReader& filteredReader, const CopyOptions& copyOptions) {
uint32_t decodeErrorCount{0};
uint32_t imageCount{0};
vector<unique_ptr<DecodeChecker>> checkers;
for (auto id : filteredReader.filter.streams) {
checkers.emplace_back(make_unique<DecodeChecker>(decodeErrorCount, imageCount));
filteredReader.reader.setStreamPlayer(id, checkers.back().get());
}

double startTimestamp, endTimestamp;
filteredReader.getConstrainedTimeRange(startTimestamp, endTimestamp);
filteredReader.preRollConfigAndState(); // make sure to copy most recent config & state records

ThrottledWriter throttledWriter(copyOptions);
throttledWriter.initTimeRange(startTimestamp, endTimestamp);

size_t readRecordCount;
bool noError;
double timeSpent = 0;
if (!iterateChecker(filteredReader, readRecordCount, noError, &throttledWriter, timeSpent)) {
return "<invalid timerange>";
}
throttledWriter.closeFile();

size_t decodedCount =
readRecordCount >= decodeErrorCount ? readRecordCount - decodeErrorCount : 0;
double successRate = decodedCount * 100. / readRecordCount;
if (copyOptions.jsonOutput) {
double duration = endTimestamp - startTimestamp;
double mbPerSec =
duration > 0 ? filteredReader.reader.getTotalSourceSize() / (duration * 1024 * 1024) : -1;
return asJson(noError, readRecordCount, duration, mbPerSec, decodedCount, successRate);
}
if (noError && decodeErrorCount == 0) {
return fmt::format(
"Decoded {} records, {} images, in {}, no errors.",
decodedCount,
imageCount,
helpers::humanReadableDuration(timeSpent));
}
return fmt::format(
"Failure! Decoded {} records out of {}, {:.2f}% good.",
decodedCount,
readRecordCount,
successRate);
}

} // namespace

namespace vrs::utils {
Expand All @@ -217,7 +354,10 @@ string checkRecords(
const CopyOptions& copyOptions,
CheckType checkType) {
if (!filteredReader.reader.isOpened()) {
return "";
return {};
}
if (checkType == CheckType::Decode) {
return decodeValidation(filteredReader, copyOptions);
}
vector<unique_ptr<RecordChecker>> checkers;
// Instance ids are assigned by VRS and can not be relied upon, though ordering is
Expand All @@ -238,7 +378,8 @@ string checkRecords(

size_t decodedCount;
bool noError;
if (!iterateChecker(filteredReader, decodedCount, noError, &throttledWriter)) {
double timeSpent = 0;
if (!iterateChecker(filteredReader, decodedCount, noError, &throttledWriter, timeSpent)) {
return "<invalid timerange>";
}
throttledWriter.closeFile();
Expand Down Expand Up @@ -293,6 +434,7 @@ string checkRecords(
} break;
case CheckType::ChecksumVerbatim:
case CheckType::HexDump:
case CheckType::Decode:
case CheckType::None:
case CheckType::Check:
case CheckType::COUNT:
Expand All @@ -316,14 +458,17 @@ string checkRecords(
duration > 0 ? filteredReader.reader.getTotalSourceSize() / (duration * 1024 * 1024) : -1;
return asJson(noError, recordCount, duration, mbPerSec, decodedCount, successRate);
}
stringstream ss;
if (noError) {
ss << "Decoded " << decodedCount << " records, no errors.";
} else {
ss << "Failure! Decoded " << decodedCount << " records out of " << recordCount << ", " << fixed
<< setprecision(2) << successRate << "% good.";
}
return ss.str();
return fmt::format(
"Checked {} records in {}, no errors.",
decodedCount,
helpers::humanReadableDuration(timeSpent));
}
return fmt::format(
"Failure! Checked {} records out of {}, {:.2f}% good.",
decodedCount,
recordCount,
successRate);
}

string recordsChecksum(const string& path, bool showProgress) {
Expand Down Expand Up @@ -643,7 +788,8 @@ bool compareVRSfiles(
first.reader.setStreamPlayer(iter.first, checkers.back().get());
}
size_t decodedCount;
if (!iterateChecker(first, decodedCount, noError, &throttledWriter)) {
double timeSpent = 0;
if (!iterateChecker(first, decodedCount, noError, &throttledWriter, timeSpent)) {
return false;
}
if (!noError) {
Expand Down
4 changes: 2 additions & 2 deletions vrs/utils/Validation.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,13 @@

namespace vrs::utils {

enum class CheckType { None, Check, Checksum, ChecksumVerbatim, Checksums, HexDump, COUNT };
enum class CheckType { None, Check, Checksum, ChecksumVerbatim, Checksums, HexDump, Decode, COUNT };

/// Check a VRS file by reading all its records & counting errors.
/// The file should be open & filters applied already.
/// @param filteredReader: source file with record filtering
/// @param copyOptions: to disable progress logging. Other parameters are not applicable.
/// @param checkType: which type of check is expected.
/// @param checkType: which type of check is expected, except ChecksumVerbatim (see API below).
std::string checkRecords(
FilteredFileReader& filteredReader,
const CopyOptions& copyOptions,
Expand Down

0 comments on commit 551235b

Please sign in to comment.