Introduce image data conversion pipeline, taking over existing YUV conversions

crates/viewer/re_renderer/shader/conversions/chroma_subsampling_converter.wgsl

@@ -0,0 +1,112 @@
+#import <../types.wgsl>
+#import <../screen_triangle_vertex.wgsl>
+
+struct UniformBuffer {
+    format: u32,
+    primaries: u32,
+    target_texture_size: vec2u,
+};
+
+@group(0) @binding(0)
+var<uniform> uniform_buffer: UniformBuffer;
+
+@group(0) @binding(1)
+var input_texture: texture_2d<u32>;
+
+
+const FORMAT_Y_UV = 0u;
+const FORMAT_YUYV16 = 1u;
+
+const PRIMARIES_BT601 = 0u;
+const PRIMARIES_BT709 = 1u;
+
+
+/// Returns sRGB from YUV color.
+///
+/// This conversion mirrors the function in `crates/store/re_types/src/datatypes/tensor_data_ext.rs`
+///
+/// Specifying the color standard should be exposed in the future [#3541](https://github.com/rerun-io/rerun/pull/3541)
+fn srgb_from_yuv(yuv: vec3f, primaries: u32) -> vec3f {
+    // rescale YUV values
+    let y = (yuv[0] - 16.0) / 219.0;
+    let u = (yuv[1] - 128.0) / 224.0;
+    let v = (yuv[2] - 128.0) / 224.0;
+
+    var rgb: vec3f;
+
+    switch (primaries) {
+        // BT.601 (aka. SDTV, aka. Rec.601). wiki: https://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.601_conversion
+        // Also note according to https://en.wikipedia.org/wiki/SRGB#sYCC_extended-gamut_transformation
+        // > Although the RGB color primaries are based on BT.709,
+        // > the equations for transformation from sRGB to sYCC and vice versa are based on BT.601.
+        case PRIMARIES_BT601: {
+            rgb.r = y + 1.402 * v;
+            rgb.g = y - 0.344 * u - 0.714 * v;
+            rgb.b = y + 1.772 * u;
+        }
+
+        // BT.709 (aka. HDTV, aka. Rec.709). wiki: https://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.709_conversion
+        case PRIMARIES_BT709: {
+            rgb.r = y + 1.575 * v;
+            rgb.g = y - 0.187 * u - 0.468 * v;
+            rgb.b = y + 1.856 * u;
+        }
+
+        default: {
+            rgb = vec3f(1.0, 0.0, 1.0); // Magenta to indicate trouble
+        }
+    }
+
+    return clamp(rgb, vec3f(0.0), vec3f(1.0));
+}
+
+/// Extracts YUV data from a chroma subsampling encoded texture at specific coordinates.
+///
+/// See also `enum ChromaSubsamplingPixelFormat` in `chroma_subsampling_converter.rs for a specification of
+/// the expected data layout.
+fn decode_chroma_subsampling_format_to_yuv(format: u32, texture: texture_2d<u32>, coords: vec2f) -> vec3f {
+    let texture_dim = vec2f(textureDimensions(texture).xy);
+    var yuv: vec3f;
+
+    switch (format)  {
+        case FORMAT_Y_UV: {
+            let uv_offset = u32(floor(texture_dim.y / 1.5));
+            let uv_row = u32(coords.y / 2);
+            var uv_col = u32(coords.x / 2) * 2u;
+
+            yuv[0] = f32(textureLoad(texture, vec2u(coords), 0).r);
+            yuv[1] = f32(textureLoad(texture, vec2u(u32(uv_col), uv_offset + uv_row), 0).r);
+            yuv[2] = f32(textureLoad(texture, vec2u((u32(uv_col) + 1u), uv_offset + uv_row), 0).r);
+        }
+
+        case FORMAT_YUYV16: {
+            // texture is 2 * width * height
+            // every 4 bytes is 2 pixels
+            let uv_row = u32(coords.y);
+            // multiply by 2 because the width is multiplied by 2
+            let y_col = u32(coords.x) * 2u;
+            yuv[0] = f32(textureLoad(texture, vec2u(y_col, uv_row), 0).r);
+
+            // at odd pixels we're in the second half of the yuyu block, offset back by 2
+            let uv_col = y_col - u32(coords.x % 2) * 2u;
+            yuv[1] = f32(textureLoad(texture, vec2u(uv_col + 1u, uv_row), 0).r);
+            yuv[2] = f32(textureLoad(texture, vec2u(uv_col + 3u, uv_row), 0).r);
+        }
+
+        default: {
+            yuv = vec3f(0.0, 0.0, 0.0);
+        }
+    }
+
+    return yuv;
+}
+
+@fragment
+fn fs_main(in: FragmentInput) -> @location(0) vec4f {
+    let coords = vec2f(uniform_buffer.target_texture_size) * in.texcoord;
+
+    let yuv = decode_chroma_subsampling_format_to_yuv(uniform_buffer.format, input_texture, coords);
+    let rgb = srgb_from_yuv(yuv, uniform_buffer.primaries);
+
+    return vec4f(rgb, 1.0);
+}

crates/viewer/re_renderer/shader/rectangle_fs.wgsl

@@ -1,11 +1,6 @@
 #import <./colormap.wgsl>
 #import <./rectangle.wgsl>
 #import <./utils/srgb.wgsl>
-#import <./decodings.wgsl>
-
-// WARNING! Adding anything else to this shader is very likely to push us over a size threshold that
-// causes the failure reported in: https://github.com/rerun-io/rerun/issues/3931
-// Make sure any changes are tested in Chrome on Linux using the Intel Mesa driver.
 
 fn is_magnifying(pixel_coord: vec2f) -> bool {
     return fwidth(pixel_coord.x) < 1.0;
@@ -86,10 +81,6 @@ fn fs_main(in: VertexOut) -> @location(0) vec4f {
         texture_dimensions = vec2f(textureDimensions(texture_sint).xy);
     } else if rect_info.sample_type == SAMPLE_TYPE_UINT {
         texture_dimensions = vec2f(textureDimensions(texture_uint).xy);
-    } else if rect_info.sample_type == SAMPLE_TYPE_NV12 {
-        texture_dimensions = vec2f(textureDimensions(texture_uint).xy);
-    } else if rect_info.sample_type == SAMPLE_TYPE_YUY2 {
-        texture_dimensions = vec2f(textureDimensions(texture_uint).xy);
     }
 
     let coord = in.texcoord * texture_dimensions;
@@ -141,14 +132,6 @@ fn fs_main(in: VertexOut) -> @location(0) vec4f {
             vec4f(textureLoad(texture_uint, v01_coord, 0)),
             vec4f(textureLoad(texture_uint, v10_coord, 0)),
             vec4f(textureLoad(texture_uint, v11_coord, 0)));
-    } else if rect_info.sample_type == SAMPLE_TYPE_NV12 || rect_info.sample_type == SAMPLE_TYPE_YUY2{
-        normalized_value = decode_color_and_filter_nearest_or_bilinear(
-            filter_nearest,
-            coord,
-            decode_nv12_or_yuy2(rect_info.sample_type, texture_uint, v00_coord),
-            decode_nv12_or_yuy2(rect_info.sample_type, texture_uint, v01_coord),
-            decode_nv12_or_yuy2(rect_info.sample_type, texture_uint, v10_coord),
-            decode_nv12_or_yuy2(rect_info.sample_type, texture_uint, v11_coord));
     } else {
         return ERROR_RGBA; // unknown sample type
     }

crates/viewer/re_renderer/src/context.rs

@@ -217,8 +217,7 @@ impl RenderContext {
         }
 
         let resolver = crate::new_recommended_file_resolver();
-        let texture_manager_2d =
-            TextureManager2D::new(device.clone(), queue.clone(), &gpu_resources.textures);
+        let texture_manager_2d = TextureManager2D::new(&device, &queue, &gpu_resources.textures);
 
         let active_frame = ActiveFrameContext {
             before_view_builder_encoder: Mutex::new(FrameGlobalCommandEncoder::new(&device)),

crates/viewer/re_renderer/src/importer/gltf.rs

@@ -8,7 +8,7 @@ use smallvec::SmallVec;
 use crate::{
     mesh::{GpuMesh, Material, Mesh, MeshError},
     renderer::MeshInstance,
-    resource_managers::{GpuTexture2D, Texture2DCreationDesc, TextureManager2D},
+    resource_managers::{GpuTexture2D, ImageDataDesc, TextureManager2D},
     RenderContext, Rgba32Unmul,
 };
 
@@ -84,31 +84,26 @@ pub fn load_gltf_from_buffer(
         #[cfg(not(debug_assertions))]
         let texture_names = "";
 
-        let texture = Texture2DCreationDesc {
+        let texture = ImageDataDesc {
             label: if texture_names.is_empty() {
                 format!("unnamed gltf image in {mesh_name}")
             } else {
                 format!("gltf image used by {texture_names} in {mesh_name}")
             }
             .into(),
             data: data.into(),
-            format,
+            format: format.into(),
             width: image.width,
             height: image.height,
         };
 
-        images_as_textures.push(
-            match ctx
-                .texture_manager_2d
-                .create(&ctx.gpu_resources.textures, &texture)
-            {
-                Ok(texture) => texture,
-                Err(err) => {
-                    re_log::error!("Failed to create texture: {err}");
-                    ctx.texture_manager_2d.white_texture_unorm_handle().clone()
-                }
-            },
-        );
+        images_as_textures.push(match ctx.texture_manager_2d.create(ctx, texture) {
+            Ok(texture) => texture,
+            Err(err) => {
+                re_log::error!("Failed to create texture: {err}");
+                ctx.texture_manager_2d.white_texture_unorm_handle().clone()
+            }
+        });
     }
 
     let mut meshes = HashMap::with_capacity(doc.meshes().len());

crates/viewer/re_renderer/src/renderer/mod.rs

@@ -82,6 +82,8 @@ pub trait Renderer {
 }
 
 /// Gets or creates a vertex shader module for drawing a screen filling triangle.
+///
+/// The entry point of this shader is `main`.
 pub fn screen_triangle_vertex_shader(
     ctx: &RenderContext,
 ) -> crate::wgpu_resources::GpuShaderModuleHandle {

crates/viewer/re_renderer/src/renderer/rectangles.rs

@@ -49,9 +49,6 @@ pub enum TextureFilterMin {
 /// Describes how the color information is encoded in the texture.
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub enum ShaderDecoding {
-    Nv12,
-    Yuy2,
-
     /// Do BGR(A)->RGB(A) conversion is in the shader.
     Bgr,
 }
@@ -145,17 +142,7 @@ impl ColormappedTexture {
     }
 
     pub fn width_height(&self) -> [u32; 2] {
-        match self.shader_decoding {
-            Some(ShaderDecoding::Nv12) => {
-                let [width, height] = self.texture.width_height();
-                [width, height * 2 / 3]
-            }
-            Some(ShaderDecoding::Yuy2) => {
-                let [width, height] = self.texture.width_height();
-                [width / 2, height]
-            }
-            Some(ShaderDecoding::Bgr) | None => self.texture.width_height(),
-        }
+        self.texture.width_height()
     }
 }
 
@@ -237,8 +224,6 @@ mod gpu_data {
     const SAMPLE_TYPE_FLOAT: u32 = 1;
     const SAMPLE_TYPE_SINT: u32 = 2;
     const SAMPLE_TYPE_UINT: u32 = 3;
-    const SAMPLE_TYPE_NV12: u32 = 4;
-    const SAMPLE_TYPE_YUY2: u32 = 5;
 
     // How do we do colormapping?
     const COLOR_MAPPER_OFF_GRAYSCALE: u32 = 1;
@@ -318,15 +303,7 @@ mod gpu_data {
             let sample_type = match texture_format.sample_type(None, None) {
                 Some(wgpu::TextureSampleType::Float { .. }) => SAMPLE_TYPE_FLOAT,
                 Some(wgpu::TextureSampleType::Sint) => SAMPLE_TYPE_SINT,
-                Some(wgpu::TextureSampleType::Uint) => {
-                    if shader_decoding == &Some(super::ShaderDecoding::Nv12) {
-                        SAMPLE_TYPE_NV12
-                    } else if shader_decoding == &Some(super::ShaderDecoding::Yuy2) {
-                        SAMPLE_TYPE_YUY2
-                    } else {
-                        SAMPLE_TYPE_UINT
-                    }
-                }
+                Some(wgpu::TextureSampleType::Uint) => SAMPLE_TYPE_UINT,
                 _ => {
                     return Err(RectangleError::TextureFormatNotSupported(texture_format));
                 }