From b9d8674204d2a7c03b735fe1e4c149351a7fe4a9 Mon Sep 17 00:00:00 2001
From: Georges Berenger <gberenger@meta.com>
Date: Thu, 26 Sep 2024 07:29:49 -0700
Subject: [PATCH] Add debayering normalization for PixelFormat::BAYER8_RGGB

Summary:
BAYER8_RGGB is currently displayed as if it was a grey8 image, which is very approximative. This diff adds a meta-only code path in which we debayer these images.
Because debayering very large images is slow, we use a debayering + downsampler algorithm for display, but with large images only.

Reviewed By: maxouellet

Differential Revision: D63151489

fbshipit-source-id: 119a047d4a0e362daa1c033caf8c90abacbc3468
---
 tools/vrsplayer/FramePlayer.cpp | 4 ++++
 vrs/utils/PixelFrame.cpp        | 4 ++++
 vrs/utils/PixelFrame.h          | 4 ++--
 3 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/tools/vrsplayer/FramePlayer.cpp b/tools/vrsplayer/FramePlayer.cpp
index fe6da072..38fad014 100644
--- a/tools/vrsplayer/FramePlayer.cpp
+++ b/tools/vrsplayer/FramePlayer.cpp
@@ -99,6 +99,7 @@ bool FramePlayer::onImageRead(
     if (frameValid) {
       normalizeOptions_ = PixelFrame::getStreamNormalizeOptions(
           *record.fileReader, record.streamId, frame->getPixelFormat());
+      normalizeOptions_.speedOverPrecision = frame->getWidth() * frame->getHeight() >= 4000 * 4000;
     }
     fmt::print(
         "Found '{} - {}': {}, {}",
@@ -235,7 +236,10 @@ bool FramePlayer::saveFrame(
     shared_ptr<PixelFrame> frame;
     if (PixelFrame::readRawFrame(frame, record.reader, spec)) {
       shared_ptr<PixelFrame> normalizedFrame;
+      bool speedOverPrecision = normalizeOptions_.speedOverPrecision;
+      normalizeOptions_.speedOverPrecision = false;
       PixelFrame::normalizeFrame(frame, normalizedFrame, true, normalizeOptions_);
+      normalizeOptions_.speedOverPrecision = speedOverPrecision;
       if (normalizedFrame->writeAsPng(saveNextFramePath_) == 0) {
         XR_LOGI("Saved raw frame as '{}'", saveNextFramePath_);
       }
diff --git a/vrs/utils/PixelFrame.cpp b/vrs/utils/PixelFrame.cpp
index 3ee88966..f406cd09 100644
--- a/vrs/utils/PixelFrame.cpp
+++ b/vrs/utils/PixelFrame.cpp
@@ -463,6 +463,10 @@ PixelFormat PixelFrame::getNormalizedPixelFormat(
        options.semantic == ImageSemantic::ObjectIdSegmentation) &&
       sourcePixelFormat == PixelFormat::GREY16) {
     format = PixelFormat::RGB8;
+#if IS_VRS_FB_INTERNAL()
+  } else if (format == PixelFormat::BAYER8_RGGB) {
+    format = PixelFormat::RGB8;
+#endif
   } else {
     format = ImageContentBlockSpec::getChannelCountPerPixel(sourcePixelFormat) > 1
         ? PixelFormat::RGB8
diff --git a/vrs/utils/PixelFrame.h b/vrs/utils/PixelFrame.h
index 86c8bf94..ade442e2 100644
--- a/vrs/utils/PixelFrame.h
+++ b/vrs/utils/PixelFrame.h
@@ -66,6 +66,7 @@ struct NormalizeOptions {
       : semantic{semantic}, min{min}, max{max} {}
 
   ImageSemantic semantic{ImageSemantic::Undefined};
+  bool speedOverPrecision{false}; // prefer speed (for display?) or precision (to save to disk?)
   float min{0};
   float max{0};
 };
@@ -73,13 +74,12 @@ struct NormalizeOptions {
 /// Helper class to read & convert images read using RecordFormat into simpler, but maybe degraded,
 /// pixel buffer, that can easily be displayed, or saved to disk as jpg or png.
 ///
-/// Here are the "conversions" performed overall:
+/// Here are some of the "normalizations" performed:
 /// - GREY10, GREY12 and GREY16 to GREY8, by pixel depth reduction.
 /// - RGB10 and RGB12 to RGB8, by pixel depth reduction.
 /// - YUV_I420_SPLIT and YUY2 to RGB8, by conversion.
 /// - DEPTH32F and SCALAR64F to GREY8, by normalization.
 ///
-/// RGB32F is not currently not supported.
 class PixelFrame {
  public:
   PixelFrame() = default;