diff --git a/crates/store/re_dataframe/src/lib.rs b/crates/store/re_dataframe/src/lib.rs index 4e05c7af09df..4dcac9edf17d 100644 --- a/crates/store/re_dataframe/src/lib.rs +++ b/crates/store/re_dataframe/src/lib.rs @@ -15,6 +15,7 @@ pub use self::external::re_chunk_store::{ }; #[doc(no_inline)] pub use self::external::re_log_types::{EntityPath, TimeInt, Timeline}; + #[doc(no_inline)] pub use self::external::re_query::Caches as QueryCache; diff --git a/crates/store/re_types/definitions/rerun/datatypes/pixel_format.fbs b/crates/store/re_types/definitions/rerun/datatypes/pixel_format.fbs index d0073e1ad2a3..91c641c72183 100644 --- a/crates/store/re_types/definitions/rerun/datatypes/pixel_format.fbs +++ b/crates/store/re_types/definitions/rerun/datatypes/pixel_format.fbs @@ -1,6 +1,7 @@ namespace rerun.datatypes; -// TODO(andreas): Clarify relationship to color primaries. Right now there's some hardcoded differences between formats. +// TODO(andreas): Clarify relationship to color primaries & yuv matrix coefficients. +// Right now there's some hardcoded differences between formats. // See `image_to_gpu.rs` // Suggestion: guides heuristic but doesn't specify it unless noted. diff --git a/crates/store/re_types/src/datatypes/pixel_format_ext.rs b/crates/store/re_types/src/datatypes/pixel_format_ext.rs index 55b12e01888b..d020d720fa92 100644 --- a/crates/store/re_types/src/datatypes/pixel_format_ext.rs +++ b/crates/store/re_types/src/datatypes/pixel_format_ext.rs @@ -1,4 +1,4 @@ -use crate::image::{rgb_from_yuv, ColorPrimaries}; +use crate::image::{rgb_from_yuv, YuvMatrixCoefficients}; use super::{ChannelDatatype, ColorModel, PixelFormat}; @@ -187,9 +187,10 @@ impl PixelFormat { } } - /// Color primaries used by this format. + /// Yuv matrix coefficients used by this format. // TODO(andreas): Expose this in the API separately and document it better. - pub fn color_primaries(&self) -> ColorPrimaries { + #[allow(clippy::unnecessary_wraps)] + pub fn yuv_matrix_coefficients(&self) -> YuvMatrixCoefficients { match self { Self::Y_U_V24_LimitedRange | Self::Y_U_V24_FullRange @@ -199,9 +200,9 @@ impl PixelFormat { | Self::Y_U_V16_FullRange // TODO(andreas): Y8 isn't really color, does this even make sense? | Self::Y8_FullRange - | Self::Y8_LimitedRange => ColorPrimaries::Bt709, + | Self::Y8_LimitedRange => YuvMatrixCoefficients::Bt709, - Self::NV12 | Self::YUY2 => ColorPrimaries::Bt601, + Self::NV12 | Self::YUY2 => YuvMatrixCoefficients::Bt601, } } @@ -216,7 +217,7 @@ impl PixelFormat { u, v, self.is_limited_yuv_range(), - self.color_primaries(), + self.yuv_matrix_coefficients(), )) } } diff --git a/crates/store/re_types/src/image.rs b/crates/store/re_types/src/image.rs index ecafce9a959d..9622a1d06b46 100644 --- a/crates/store/re_types/src/image.rs +++ b/crates/store/re_types/src/image.rs @@ -266,13 +266,22 @@ fn test_find_non_empty_dim_indices() { // ---------------------------------------------------------------------------- -// TODO(andreas): Expose this in the API. -/// Type of color primaries a given image is in. +// TODO(andreas): Expose this in the API? +/// Yuv matrix coefficients that determine how a YUV image is meant to be converted to RGB. /// -/// This applies both to YUV and RGB formats, but if not specified otherwise -/// we assume BT.709 primaries for all RGB(A) 8bits per channel content. +/// A rigorious definition of the yuv conversion matrix would still require to define +/// the transfer characteristics & color primaries of the resulting RGB space. +/// See [`re_video::decode`]'s documentation. +/// +/// However, at this point we generally assume that no further processing is needed after the transform. +/// This is acceptable for most non-HDR content because of the following properties of `Bt709`/`Bt601`/ sRGB: +/// * Bt709 & sRGB primaries are practically identical +/// * Bt601 PAL & Bt709 color primaries are the same (with some slight differences for Bt709 NTSC) +/// * Bt709 & sRGB transfer function are almost identical (and the difference is widely ignored) +/// (sources: , ) +/// …which means for the moment we pretty much only care about the (actually quite) different YUV conversion matrices! #[derive(Clone, Copy, Debug)] -pub enum ColorPrimaries { +pub enum YuvMatrixCoefficients { /// BT.601 (aka. SDTV, aka. Rec.601) /// /// Wiki: @@ -310,7 +319,7 @@ pub fn rgb_from_yuv( u: u8, v: u8, limited_range: bool, - primaries: ColorPrimaries, + coefficients: YuvMatrixCoefficients, ) -> [u8; 3] { let (mut y, mut u, mut v) = (y as f32, u as f32, v as f32); @@ -332,15 +341,15 @@ pub fn rgb_from_yuv( let g; let b; - match primaries { - ColorPrimaries::Bt601 => { + match coefficients { + YuvMatrixCoefficients::Bt601 => { // BT.601 (aka. SDTV, aka. Rec.601). wiki: https://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.601_conversion r = y + 1.402 * v; g = y - 0.344 * u - 0.714 * v; b = y + 1.772 * u; } - ColorPrimaries::Bt709 => { + YuvMatrixCoefficients::Bt709 => { // BT.709 (aka. HDTV, aka. Rec.709). wiki: https://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.709_conversion r = y + 1.575 * v; g = y - 0.187 * u - 0.468 * v; diff --git a/crates/store/re_video/src/decode/av1.rs b/crates/store/re_video/src/decode/av1.rs index 61c29e2a0f4f..d3982b3be503 100644 --- a/crates/store/re_video/src/decode/av1.rs +++ b/crates/store/re_video/src/decode/av1.rs @@ -6,7 +6,7 @@ use crate::Time; use dav1d::{PixelLayout, PlanarImageComponent}; use super::{ - Chunk, ColorPrimaries, Error, Frame, OutputCallback, PixelFormat, Result, SyncDecoder, + Chunk, Error, Frame, OutputCallback, PixelFormat, Result, SyncDecoder, YuvMatrixCoefficients, YuvPixelLayout, YuvRange, }; @@ -115,11 +115,6 @@ fn output_picture( picture: &dav1d::Picture, on_output: &(dyn Fn(Result) + Send + Sync), ) { - // TODO(jan): support other parameters? - // What do these even do: - // - matrix_coefficients - // - transfer_characteristics - let data = { re_tracing::profile_scope!("copy_picture_data"); @@ -218,7 +213,7 @@ fn output_picture( dav1d::pixel::YUVRange::Limited => YuvRange::Limited, dav1d::pixel::YUVRange::Full => YuvRange::Full, }, - primaries: color_primaries(debug_name, picture), + coefficients: yuv_matrix_coefficients(debug_name, picture), }; let frame = Frame { @@ -232,68 +227,76 @@ fn output_picture( on_output(Ok(frame)); } -fn color_primaries(debug_name: &str, picture: &dav1d::Picture) -> ColorPrimaries { +fn yuv_matrix_coefficients(debug_name: &str, picture: &dav1d::Picture) -> YuvMatrixCoefficients { + // Quotes are from https://wiki.x266.mov/docs/colorimetry/matrix (if not noted otherwise) #[allow(clippy::match_same_arms)] - match picture.color_primaries() { - dav1d::pixel::ColorPrimaries::Reserved - | dav1d::pixel::ColorPrimaries::Reserved0 - | dav1d::pixel::ColorPrimaries::Unspecified => { + match picture.matrix_coefficients() { + // TODO(andreas) This one we should probably support! Afaik this means to just interpret YUV as RGB (or is there a swizzle?). + dav1d::pixel::MatrixCoefficients::Identity => YuvMatrixCoefficients::Bt709, + + dav1d::pixel::MatrixCoefficients::BT709 => YuvMatrixCoefficients::Bt709, + + dav1d::pixel::MatrixCoefficients::Unspecified + | dav1d::pixel::MatrixCoefficients::Reserved => { // This happens quite often. Don't issue a warning, that would be noise! if picture.transfer_characteristic() == dav1d::pixel::TransferCharacteristic::SRGB { // If the transfer characteristic is sRGB, assume BT.709 primaries, would be quite odd otherwise. // TODO(andreas): Other transfer characteristics may also hint at primaries. - ColorPrimaries::Bt709 + YuvMatrixCoefficients::Bt709 } else { // Best guess: If the picture is 720p+ assume Bt709 because Rec709 // is the "HDR" standard. // TODO(#7594): 4k/UHD material should probably assume Bt2020? // else if picture.height() >= 720 { - // ColorPrimaries::Bt709 + // YuvMatrixCoefficients::Bt709 // } else { - // ColorPrimaries::Bt601 + // YuvMatrixCoefficients::Bt601 // } // // This is also what the mpv player does (and probably others): - // https://wiki.x266.mov/docs/colorimetry/primaries#2-unspecified + // https://wiki.x266.mov/docs/colorimetry/matrix#2-unspecified + // (and similar for primaries! https://wiki.x266.mov/docs/colorimetry/primaries#2-unspecified) // // …then again, eyeballing VLC it looks like it just always assumes BT.709. // The handwavy test case employed here was the same video in low & high resolution // without specified primaries. Both looked the same. - ColorPrimaries::Bt709 + YuvMatrixCoefficients::Bt709 } } - dav1d::pixel::ColorPrimaries::BT709 => ColorPrimaries::Bt709, - - // NTSC standard. Close enough to BT.601 for now. TODO(andreas): Is it worth warning? - dav1d::pixel::ColorPrimaries::BT470M => ColorPrimaries::Bt601, - - // PAL standard. Close enough to BT.601 for now. TODO(andreas): Is it worth warning? - dav1d::pixel::ColorPrimaries::BT470BG => ColorPrimaries::Bt601, - - // These are both using BT.2020 primaries. - dav1d::pixel::ColorPrimaries::ST170M | dav1d::pixel::ColorPrimaries::ST240M => { - ColorPrimaries::Bt601 + dav1d::pixel::MatrixCoefficients::BT470M => { + // "BT.470M is a standard that was used in analog television systems in the United States." + // I guess Bt601 will do! + YuvMatrixCoefficients::Bt601 + } + dav1d::pixel::MatrixCoefficients::BT470BG | dav1d::pixel::MatrixCoefficients::ST170M => { + // This is PAL & NTSC standards, both are part of Bt.601. + YuvMatrixCoefficients::Bt601 + } + dav1d::pixel::MatrixCoefficients::ST240M => { + // "SMPTE 240M was an interim standard used during the early days of HDTV (1988-1998)." + // Not worth the effort: HD -> Bt709 🤷 + YuvMatrixCoefficients::Bt709 } - // Is st428 also HDR? Not sure. - // BT2020 and P3 variants definitely are ;) - dav1d::pixel::ColorPrimaries::BT2020 - | dav1d::pixel::ColorPrimaries::ST428 - | dav1d::pixel::ColorPrimaries::P3DCI - | dav1d::pixel::ColorPrimaries::P3Display => { - // TODO(#7594): HDR support. + dav1d::pixel::MatrixCoefficients::BT2020NonConstantLuminance + | dav1d::pixel::MatrixCoefficients::BT2020ConstantLuminance + | dav1d::pixel::MatrixCoefficients::ICtCp + | dav1d::pixel::MatrixCoefficients::ST2085 => { + // TODO(#7594): HDR support (we'll probably only care about `BT2020NonConstantLuminance`?) re_log::warn_once!("Video {debug_name:?} specified HDR color primaries. Rerun doesn't handle HDR colors correctly yet. Color artifacts may be visible."); - ColorPrimaries::Bt709 + YuvMatrixCoefficients::Bt709 } - dav1d::pixel::ColorPrimaries::Film | dav1d::pixel::ColorPrimaries::Tech3213 => { + dav1d::pixel::MatrixCoefficients::ChromaticityDerivedNonConstantLuminance + | dav1d::pixel::MatrixCoefficients::ChromaticityDerivedConstantLuminance + | dav1d::pixel::MatrixCoefficients::YCgCo => { re_log::warn_once!( - "Video {debug_name:?} specified unsupported color primaries {:?}. Color artifacts may be visible.", - picture.color_primaries() - ); - ColorPrimaries::Bt709 + "Video {debug_name:?} specified unsupported matrix coefficients {:?}. Color artifacts may be visible.", + picture.matrix_coefficients() + ); + YuvMatrixCoefficients::Bt709 } } } diff --git a/crates/store/re_video/src/decode/mod.rs b/crates/store/re_video/src/decode/mod.rs index 77f641b41db9..b938836f2e3f 100644 --- a/crates/store/re_video/src/decode/mod.rs +++ b/crates/store/re_video/src/decode/mod.rs @@ -1,4 +1,82 @@ //! Video frame decoding. +//! ========================= +//! +//! Whirlwind tour of how to interpret picture data (from a Video perspective) +//! --------------------------------------------------------------------------------- +//! +//! Extracted from the [av1 codec wiki](https://wiki.x266.mov/docs/colorimetry/intro) and other sources. +//! Follows the trail of information we get from our AV1 decoder. +//! +//! ### How to get from YUV to RGB? +//! +//! Things to know about the incoming yuv data: +//! * `picture.bit_depth()` +//! * is either 8 or 16 +//! * that's how the decoder stores for us but the per component we have either 8 or 10 or 12 bits -> see `picture.bits_per_component()` +//! * `picture.pixel_layout()` +//! * `4:0:0` greyscale +//! * `4:2:0` half horizontal and half vertical resolution for chroma +//! * `4:2:2` half horizontal resolution for chroma +//! * `4:4:4` full resolution for chroma +//! * note that the AV1 decoder gives us always (!) planar data +//! * `picture.color_range()` +//! * yuv data range may be either `limited` or `full` +//! * `full` is what you'd naively expect, just full use up the entire 8/10/12 bits! +//! * `limited` means that only a certain range of values is valid +//! * weirdly enough, DO NOT CLAMP! a lot of software may say it's limited but then use the so-called foot and head space anyways to go outside the regular colors +//! * reportedly (read this on some forums ;-)) some players _do_ clamp, so let's not get too concerned about this +//! * it's a remnant of the analog age, but it's still very common! +//! * TODO(andreas): It may actually be non-downsampled RGB as well, so skip the entire YUV conversion step! Haven't figured out yet how to determine that. +//! +//! +//! ### Given a normalized YUV triplet, how do we get color? +//! +//! * `picture.matrix_coefficients()` (see ) +//! * this tells us what to multiply the incoming YUV data with to get SOME RGB data +//! * there's various standards of how to do this, but the most common is BT.709 +//! * `picture.primaries()` +//! * now we have RGB but we kinda have no idea what that means! +//! * the color primaries tell us which space we're in +//! * ...meaning that if the primaries are anything else we'd have to do some conversion BUT +//! it also means that we have no chance of displaying the picture perfectly on a screen taking in sRGB (or any other not-matching color space) +//! * [Wikipedia says](https://en.wikipedia.org/wiki/Rec._709#Relationship_to_sRGB) sRGB uses the same primaries as BT.709 +//! * but I also found other sources (e.g. [this forum post](https://forum.doom9.org/showthread.php?p=1640342#post1640342)) +//! clamining that they're just close enough to be considered the same for practical purposes +//! * `picture.transfer_characteristics()` +//! * until this point everything is "gamma compressed", or more accurately, went through Opto Electric Transfer Function (OETF) +//! * i.e. measure of light in, electronic signal out +//! * we have to keep in mind the EOTF that our screen at the other end will use which for today's renderpipeline is always sRGB +//! (meaning it's a 2.2 gamma curve with a small linear part) +//! * Similar to the primaries, BT.709 uses a _similar_ transfer function as sRGB, but not exactly the same +//! +//! * There's reason to believe players just ignore this: +//! * From a [VLC issue](https://code.videolan.org/videolan/vlc/-/issues/26999): +//! > We do not support transfers or primaries anyway, so it does not matter +//! > (we do support HDR transfer functions PQ and HLG, not SDR ones and we support BT.2020 primaries, but not SMPTE C (which is what BT.601 NTSC is))." +//! * …I'm sure I found a report of other video players ignoring this and most of everything except `matrix_coefficients` but I can't find it anymore :( +//! +//! All of the above are completely optional for a video to specify and there's sometimes some interplay of relationships with those. +//! (a standard would often specify several things at once, there's typical and less typical combinations) +//! So naturally, people will use terms sloppily and interchangeably, +//! If anything is lacking a video player has to make a guess. +//! … and as discussed above, even it's there, often video players tend to ignore some settings! +//! +//! With all this out of the way… +//! +//! ### What's the state of us making use of all these things? +//! +//! * ❌ `picture.bit_depth()` +//! * TODO(#7594): ignored, we just pretend everything is 8 bits +//! * ✅ `picture.pixel_layout()` +//! * ✅ `picture.color_range()` +//! * 🟧 `picture.matrix_coefficients()` +//! * we try to figure out whether to use `BT.709` or `BT.601` coefficients, using other characteristics for guessing if nothing else is available. +//! * ❌ `picture.primaries()` +//! * ❌ `picture.transfer_characteristics()` +//! +//! We'll very likely be good with this until either we get specific feature requests and/or we'll start +//! supporting HDR content at which point more properties will be important! +//! #[cfg(feature = "av1")] #[cfg(not(target_arch = "wasm32"))] @@ -70,7 +148,7 @@ pub enum PixelFormat { range: YuvRange, // TODO(andreas): color primaries should also apply to RGB data, // but for now we just always assume RGB to be BT.709 ~= sRGB. - primaries: ColorPrimaries, + coefficients: YuvMatrixCoefficients, }, } @@ -95,11 +173,11 @@ pub enum YuvRange { Full, } -/// Color primaries used by [`PixelFormat::Yuv`]. +/// Yuv matrix coefficients used by [`PixelFormat::Yuv`]. /// -/// For details see `re_renderer`'s `ColorPrimaries` type. +/// For details see `re_renderer`'s `YuvMatrixCoefficients` type. #[derive(Debug)] -pub enum ColorPrimaries { +pub enum YuvMatrixCoefficients { Bt601, Bt709, } diff --git a/crates/viewer/re_renderer/shader/conversions/yuv_converter.wgsl b/crates/viewer/re_renderer/shader/conversions/yuv_converter.wgsl index 29e81a5030f2..731c632dcb62 100644 --- a/crates/viewer/re_renderer/shader/conversions/yuv_converter.wgsl +++ b/crates/viewer/re_renderer/shader/conversions/yuv_converter.wgsl @@ -3,7 +3,7 @@ struct UniformBuffer { yuv_layout: u32, - primaries: u32, + yuv_matrix_coefficients: u32, target_texture_size: vec2u, yuv_range: u32, @@ -24,9 +24,9 @@ const YUV_LAYOUT_Y_UV420 = 100u; const YUV_LAYOUT_YUYV422 = 200u; const YUV_LAYOUT_Y400 = 300u; -// see `enum ColorPrimaries`. -const PRIMARIES_BT601 = 0u; -const PRIMARIES_BT709 = 1u; +// see `enum YuvMatrixCoefficients`. +const COEFFS_BT601 = 0u; +const COEFFS_BT709 = 1u; // see `enum YuvRange`. const YUV_RANGE_LIMITED = 0u; @@ -38,7 +38,7 @@ const YUV_RANGE_FULL = 1u; /// This conversion mirrors the function in `crates/store/re_types/src/datatypes/tensor_data_ext.rs` /// /// Specifying the color standard should be exposed in the future [#3541](https://github.com/rerun-io/rerun/pull/3541) -fn srgb_from_yuv(yuv: vec3f, primaries: u32, range: u32) -> vec3f { +fn srgb_from_yuv(yuv: vec3f, yuv_matrix_coefficients: u32, range: u32) -> vec3f { // rescale YUV values // // This is what is called "limited range" and is the most common case. @@ -72,19 +72,19 @@ fn srgb_from_yuv(yuv: vec3f, primaries: u32, range: u32) -> vec3f { var rgb: vec3f; - switch (primaries) { + switch (yuv_matrix_coefficients) { // BT.601 (aka. SDTV, aka. Rec.601). wiki: https://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.601_conversion // Also note according to https://en.wikipedia.org/wiki/SRGB#sYCC_extended-gamut_transformation // > Although the RGB color primaries are based on BT.709, // > the equations for transformation from sRGB to sYCC and vice versa are based on BT.601. - case PRIMARIES_BT601: { + case COEFFS_BT601: { rgb.r = y + 1.402 * v; rgb.g = y - 0.344 * u - 0.714 * v; rgb.b = y + 1.772 * u; } // BT.709 (aka. HDTV, aka. Rec.709). wiki: https://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.709_conversion - case PRIMARIES_BT709: { + case COEFFS_BT709: { rgb.r = y + 1.575 * v; rgb.g = y - 0.187 * u - 0.468 * v; rgb.b = y + 1.856 * u; @@ -187,7 +187,7 @@ fn fs_main(in: FragmentInput) -> @location(0) vec4f { let coords = vec2u(vec2f(uniform_buffer.target_texture_size) * in.texcoord); let yuv = sample_yuv(uniform_buffer.yuv_layout, input_texture, coords, uniform_buffer.target_texture_size); - let rgb = srgb_from_yuv(yuv, uniform_buffer.primaries, uniform_buffer.yuv_range); + let rgb = srgb_from_yuv(yuv, uniform_buffer.yuv_matrix_coefficients, uniform_buffer.yuv_range); return vec4f(rgb, 1.0); } diff --git a/crates/viewer/re_renderer/src/resource_managers/image_data_to_texture.rs b/crates/viewer/re_renderer/src/resource_managers/image_data_to_texture.rs index b08de108fa0f..939b441c217a 100644 --- a/crates/viewer/re_renderer/src/resource_managers/image_data_to_texture.rs +++ b/crates/viewer/re_renderer/src/resource_managers/image_data_to_texture.rs @@ -1,50 +1,14 @@ -use super::yuv_converter::{YuvFormatConversionTask, YuvPixelLayout, YuvRange}; +//! For an overview of image data interpretation check `re_video`'s decoder docs! + +use super::yuv_converter::{ + YuvFormatConversionTask, YuvMatrixCoefficients, YuvPixelLayout, YuvRange, +}; use crate::{ renderer::DrawError, wgpu_resources::{GpuTexture, TextureDesc}, DebugLabel, RenderContext, Texture2DBufferInfo, }; -/// Type of color primaries a given image is in. -/// -/// This applies both to YUV and RGB formats, but if not specified otherwise -/// we assume BT.709 primaries for all RGB(A) 8bits per channel content (details below on [`ColorPrimaries::Bt709`]). -/// Since with YUV content the color space is often less clear, we always explicitly -/// specify it. -/// -/// Ffmpeg's documentation has a short & good overview of the relationship of YUV & color primaries: -/// -/// -/// Values need to be kept in sync with `yuv_converter.wgsl` -#[derive(Clone, Copy, Debug)] -pub enum ColorPrimaries { - /// BT.601 (aka. SDTV, aka. Rec.601) - /// - /// Wiki: - Bt601 = 0, - - /// BT.709 (aka. HDTV, aka. Rec.709) - /// - /// Wiki: - /// - /// These are the same primaries we usually assume and use for all our rendering - /// since they are the same primaries used by sRGB. - /// - /// The OETF/EOTF function () is different, - /// but for all other purposes they are the same. - /// (The only reason for us to convert to optical units ("linear" instead of "gamma") is for - /// lighting & tonemapping where we typically start out with an sRGB image!) - Bt709 = 1, - // - // Not yet supported. These vary a lot more from the other two! - // - // /// BT.2020 (aka. PQ, aka. Rec.2020) - // /// - // /// Wiki: - // BT2020_ConstantLuminance, - // BT2020_NonConstantLuminance, -} - /// Image data format that can be converted to a wgpu texture. // TODO(andreas): Right now this combines both color space and pixel format. Consider separating them similar to how we do on user facing APIs. #[allow(non_camel_case_types)] @@ -61,7 +25,7 @@ pub enum SourceImageDataFormat { /// YUV (== `YCbCr`) formats, typically using chroma downsampling. Yuv { layout: YuvPixelLayout, - primaries: ColorPrimaries, + coefficients: YuvMatrixCoefficients, range: YuvRange, }, // @@ -351,13 +315,13 @@ pub fn transfer_image_data_to_texture( } SourceImageDataFormat::Yuv { layout, - primaries, + coefficients, range, } => YuvFormatConversionTask::new( ctx, layout, range, - primaries, + coefficients, &data_texture, target_texture, ), @@ -420,4 +384,4 @@ fn copy_data_to_texture( .copy_to_texture2d_entire_first_layer(before_view_builder_encoder.get(), data_texture)?; Ok(()) -} +} \ No newline at end of file diff --git a/crates/viewer/re_renderer/src/resource_managers/mod.rs b/crates/viewer/re_renderer/src/resource_managers/mod.rs index 5f62146856b8..3650190f572e 100644 --- a/crates/viewer/re_renderer/src/resource_managers/mod.rs +++ b/crates/viewer/re_renderer/src/resource_managers/mod.rs @@ -11,8 +11,7 @@ mod texture_manager; mod yuv_converter; pub use image_data_to_texture::{ - transfer_image_data_to_texture, ColorPrimaries, ImageDataDesc, ImageDataToTextureError, - SourceImageDataFormat, + transfer_image_data_to_texture, ImageDataDesc, ImageDataToTextureError, SourceImageDataFormat, }; pub use texture_manager::{GpuTexture2D, TextureManager2D, TextureManager2DError}; -pub use yuv_converter::{YuvPixelLayout, YuvRange}; +pub use yuv_converter::{YuvMatrixCoefficients, YuvPixelLayout, YuvRange}; diff --git a/crates/viewer/re_renderer/src/resource_managers/yuv_converter.rs b/crates/viewer/re_renderer/src/resource_managers/yuv_converter.rs index 9aea41c22b29..bd0237bac2ff 100644 --- a/crates/viewer/re_renderer/src/resource_managers/yuv_converter.rs +++ b/crates/viewer/re_renderer/src/resource_managers/yuv_converter.rs @@ -11,8 +11,6 @@ use crate::{ RenderContext, }; -use super::ColorPrimaries; - /// Supported chroma subsampling input formats. /// /// We use `YUV`/`YCbCr`/`YPbPr` interchangeably and usually just call it `YUV`. @@ -174,6 +172,48 @@ pub enum YuvPixelLayout { Y400 = 300, } +/// Yuv matrix coefficients that determine how a YUV image is meant to be converted to RGB. +/// +/// A rigorious definition of the yuv conversion matrix would still require to define +/// the transfer characteristics & color primaries of the resulting RGB space. +/// See [`re_video::decode`]'s documentation. +/// +/// However, at this point we generally assume that no further processing is needed after the transform. +/// This is acceptable for most non-HDR content because of the following properties of `Bt709`/`Bt601`/ sRGB: +/// * Bt709 & sRGB primaries are practically identical +/// * Bt601 PAL & Bt709 color primaries are the same (with some slight differences for Bt709 NTSC) +/// * Bt709 & sRGB transfer function are almost identical (and the difference is widely ignored) +/// (sources: , ) +/// …which means for the moment we pretty much only care about the (actually quite) different YUV conversion matrices! +#[derive(Clone, Copy, Debug)] +pub enum YuvMatrixCoefficients { + /// BT.601 (aka. SDTV, aka. Rec.601) + /// + /// Wiki: + Bt601 = 0, + + /// BT.709 (aka. HDTV, aka. Rec.709) + /// + /// Wiki: + /// + /// These are the same primaries we usually assume and use for all our rendering + /// since they are the same primaries used by sRGB. + /// + /// The OETF/EOTF function () is different, + /// but for all other purposes they are the same. + /// (The only reason for us to convert to optical units ("linear" instead of "gamma") is for + /// lighting & tonemapping where we typically start out with an sRGB image!) + Bt709 = 1, + // + // Not yet supported. These vary a lot more from the other two! + // + // /// BT.2020 (aka. PQ, aka. Rec.2020) + // /// + // /// Wiki: + // BT2020_ConstantLuminance, + // BT2020_NonConstantLuminance, +} + /// Expected range of YUV values. /// /// Keep indices in sync with `yuv_converter.wgsl` @@ -253,8 +293,8 @@ mod gpu_data { /// Uses [`super::YuvPixelLayout`]. pub yuv_layout: u32, - /// Uses [`super::ColorPrimaries`]. - pub primaries: u32, + /// Uses [`super::YuvMatrixCoefficients`]. + pub yuv_matrix_coefficients: u32, pub target_texture_size: [u32; 2], @@ -296,7 +336,7 @@ impl YuvFormatConversionTask { ctx: &RenderContext, yuv_layout: YuvPixelLayout, yuv_range: YuvRange, - primaries: ColorPrimaries, + yuv_matrix_coefficients: YuvMatrixCoefficients, input_data: &GpuTexture, target_texture: &GpuTexture, ) -> Self { @@ -308,7 +348,7 @@ impl YuvFormatConversionTask { format!("{target_label}_conversion").into(), gpu_data::UniformBuffer { yuv_layout: yuv_layout as _, - primaries: primaries as _, + yuv_matrix_coefficients: yuv_matrix_coefficients as _, target_texture_size: [ target_texture.creation_desc.size.width, target_texture.creation_desc.size.height, diff --git a/crates/viewer/re_renderer/src/video/decoder/native_decoder.rs b/crates/viewer/re_renderer/src/video/decoder/native_decoder.rs index fccdb1fe13ec..b99385e2f51e 100644 --- a/crates/viewer/re_renderer/src/video/decoder/native_decoder.rs +++ b/crates/viewer/re_renderer/src/video/decoder/native_decoder.rs @@ -8,8 +8,8 @@ use parking_lot::Mutex; use crate::{ resource_managers::{ - transfer_image_data_to_texture, ColorPrimaries, ImageDataDesc, SourceImageDataFormat, - YuvPixelLayout, YuvRange, + transfer_image_data_to_texture, ImageDataDesc, SourceImageDataFormat, + YuvMatrixCoefficients, YuvPixelLayout, YuvRange, }, video::DecodingError, wgpu_resources::GpuTexture, @@ -169,7 +169,7 @@ fn copy_video_frame_to_texture( re_video::PixelFormat::Yuv { layout, range, - primaries, + coefficients, } => SourceImageDataFormat::Yuv { layout: match layout { re_video::decode::YuvPixelLayout::Y_U_V444 => YuvPixelLayout::Y_U_V444, @@ -177,9 +177,9 @@ fn copy_video_frame_to_texture( re_video::decode::YuvPixelLayout::Y_U_V420 => YuvPixelLayout::Y_U_V420, re_video::decode::YuvPixelLayout::Y400 => YuvPixelLayout::Y400, }, - primaries: match primaries { - re_video::decode::ColorPrimaries::Bt601 => ColorPrimaries::Bt601, - re_video::decode::ColorPrimaries::Bt709 => ColorPrimaries::Bt709, + coefficients: match coefficients { + re_video::decode::YuvMatrixCoefficients::Bt601 => YuvMatrixCoefficients::Bt601, + re_video::decode::YuvMatrixCoefficients::Bt709 => YuvMatrixCoefficients::Bt709, }, range: match range { re_video::decode::YuvRange::Limited => YuvRange::Limited, diff --git a/crates/viewer/re_viewer_context/src/gpu_bridge/image_to_gpu.rs b/crates/viewer/re_viewer_context/src/gpu_bridge/image_to_gpu.rs index 41f5c2e3983f..cac1dadabeae 100644 --- a/crates/viewer/re_viewer_context/src/gpu_bridge/image_to_gpu.rs +++ b/crates/viewer/re_viewer_context/src/gpu_bridge/image_to_gpu.rs @@ -11,7 +11,7 @@ use re_renderer::{ pad_rgb_to_rgba, renderer::{ColorMapper, ColormappedTexture, ShaderDecoding}, resource_managers::{ - ColorPrimaries, ImageDataDesc, SourceImageDataFormat, YuvPixelLayout, YuvRange, + ImageDataDesc, SourceImageDataFormat, YuvMatrixCoefficients, YuvPixelLayout, YuvRange, }, RenderContext, }; @@ -248,9 +248,9 @@ pub fn texture_creation_desc_from_color_image<'a>( let (data, format) = if let Some(pixel_format) = image.format.pixel_format { let data = cast_slice_to_cow(image.buffer.as_slice()); - let primaries = match pixel_format.color_primaries() { - re_types::image::ColorPrimaries::Bt601 => ColorPrimaries::Bt601, - re_types::image::ColorPrimaries::Bt709 => ColorPrimaries::Bt709, + let coefficients = match pixel_format.yuv_matrix_coefficients() { + re_types::image::YuvMatrixCoefficients::Bt601 => YuvMatrixCoefficients::Bt601, + re_types::image::YuvMatrixCoefficients::Bt709 => YuvMatrixCoefficients::Bt709, }; let range = match pixel_format.is_limited_yuv_range() { @@ -262,14 +262,14 @@ pub fn texture_creation_desc_from_color_image<'a>( // For historical reasons, using Bt.709 for fully planar formats and Bt.601 for others. // // TODO(andreas): Investigate if there's underlying expectation for some of these (for instance I suspect that NV12 is "usually" BT601). - // TODO(andreas): Expose color primaries. It's probably still the better default (for instance that's what jpeg still uses), + // TODO(andreas): Expose coefficients. It's probably still the better default (for instance that's what jpeg still uses), // but should confirm & back that up! // PixelFormat::Y_U_V24_FullRange | PixelFormat::Y_U_V24_LimitedRange => { SourceImageDataFormat::Yuv { layout: YuvPixelLayout::Y_U_V444, range, - primaries, + coefficients, } } @@ -277,7 +277,7 @@ pub fn texture_creation_desc_from_color_image<'a>( SourceImageDataFormat::Yuv { layout: YuvPixelLayout::Y_U_V422, range, - primaries, + coefficients, } } @@ -285,7 +285,7 @@ pub fn texture_creation_desc_from_color_image<'a>( SourceImageDataFormat::Yuv { layout: YuvPixelLayout::Y_U_V420, range, - primaries, + coefficients, } } @@ -293,20 +293,20 @@ pub fn texture_creation_desc_from_color_image<'a>( SourceImageDataFormat::Yuv { layout: YuvPixelLayout::Y400, range, - primaries, + coefficients, } } PixelFormat::NV12 => SourceImageDataFormat::Yuv { layout: YuvPixelLayout::Y_UV420, range, - primaries, + coefficients, }, PixelFormat::YUY2 => SourceImageDataFormat::Yuv { layout: YuvPixelLayout::YUYV422, range, - primaries, + coefficients, }, }; diff --git a/crates/viewer/re_viewer_context/src/image_info.rs b/crates/viewer/re_viewer_context/src/image_info.rs index 6a19d2f0876f..913f313bd72c 100644 --- a/crates/viewer/re_viewer_context/src/image_info.rs +++ b/crates/viewer/re_viewer_context/src/image_info.rs @@ -104,7 +104,7 @@ impl ImageInfo { u, v, pixel_format.is_limited_yuv_range(), - pixel_format.color_primaries(), + pixel_format.yuv_matrix_coefficients(), ); Some(TensorElement::U8(rgb[channel as usize])) } else if channel == 4 { diff --git a/tests/python/release_checklist/check_chroma_subsampling.py b/tests/python/release_checklist/check_chroma_subsampling.py index 4cfda9ad5d6c..c072c57a995e 100644 --- a/tests/python/release_checklist/check_chroma_subsampling.py +++ b/tests/python/release_checklist/check_chroma_subsampling.py @@ -17,7 +17,7 @@ # Chroma subsampling All images should look roughly the same except for some chroma artifacts -and slight color differences due to different color primaries. +and slight color differences due to different yuv conversion matrix coefficients. Naturally, Y8 formats are greyscale. """