From 26ccb8337e33f1146d55a1fbb7e1730f980aacaf Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Thu, 17 Oct 2024 12:13:39 +0200 Subject: [PATCH] New cross-language example snippet better demonstrating image formats (#7785) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### What ![full](https://github.com/user-attachments/assets/70208860-ee0c-4196-aa2c-860e39dbe889) * Fixes #7691 Also unearthed a snippet that showed use with PIL & OpenCV which we didn't reference so far 😱 . Overlapping, but still useful on its own, so I put it into our docs as well! ### Checklist * [x] pass main ci * [x] I have read and agree to [Contributor Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and the [Code of Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md) * [x] I've included a screenshot or gif (if applicable) * [x] I have tested the web demo (if applicable): * Using examples from latest `main` build: [rerun.io/viewer](https://rerun.io/viewer/pr/7785?manifest_url=https://app.rerun.io/version/main/examples_manifest.json) * Using full set of examples from `nightly` build: [rerun.io/viewer](https://rerun.io/viewer/pr/7785?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json) * [x] The PR title and labels are set such as to maximize their usefulness for the next release's CHANGELOG * [x] If applicable, add a new check to the [release checklist](https://github.com/rerun-io/rerun/blob/main/tests/python/release_checklist)! * [x] If have noted any breaking changes to the log API in `CHANGELOG.md` and the migration guide - [PR Build Summary](https://build.rerun.io/pr/7785) - [Recent benchmark results](https://build.rerun.io/graphs/crates.html) - [Wasm size tracking](https://build.rerun.io/graphs/sizes.html) To run all checks from `main`, comment on the PR with `@rerun-bot full-check`. --- .../definitions/rerun/archetypes/image.fbs | 3 +- crates/store/re_types/src/archetypes/image.rs | 90 +++++++++---------- .../reference/types/archetypes/image.md | 16 ++-- .../snippets/all/archetypes/image_advanced.py | 6 +- .../snippets/all/archetypes/image_formats.cpp | 64 +++++++++++++ docs/snippets/all/archetypes/image_formats.py | 18 ++++ docs/snippets/all/archetypes/image_formats.rs | 53 +++++++++++ rerun_cpp/src/rerun.hpp | 1 + rerun_cpp/src/rerun/archetypes/image.hpp | 88 ++++++++++-------- rerun_py/rerun_sdk/rerun/archetypes/image.py | 48 ++++------ 10 files changed, 261 insertions(+), 126 deletions(-) create mode 100644 docs/snippets/all/archetypes/image_formats.cpp create mode 100644 docs/snippets/all/archetypes/image_formats.py create mode 100644 docs/snippets/all/archetypes/image_formats.rs diff --git a/crates/store/re_types/definitions/rerun/archetypes/image.fbs b/crates/store/re_types/definitions/rerun/archetypes/image.fbs index 39684dd9dc5c..0617aac1d6ce 100644 --- a/crates/store/re_types/definitions/rerun/archetypes/image.fbs +++ b/crates/store/re_types/definitions/rerun/archetypes/image.fbs @@ -21,7 +21,8 @@ namespace rerun.archetypes; /// \cpp If needed, this "borrow-behavior" can be extended by defining your own `rerun::CollectionAdapter`. /// /// \example archetypes/image_simple image="https://static.rerun.io/image_simple/06ba7f8582acc1ffb42a7fd0006fad7816f3e4e4/1200w.png" -/// \example archetypes/image_send_columns title="Advanced usage of `send_columns` to send multiple images at once" image="https://static.rerun.io/image_send_columns/321455161d79e2c45d6f5a6f175d6f765f418897/1200w.png" +/// \example archetypes/image_formats title="Logging images with various formats" image="https://static.rerun.io/image_formats/7b8a162fcfd266f303980439beea997dc8544c24/full.png" +/// \example archetypes/image_send_columns !api title="Image from file, PIL & OpenCV" image="https://static.rerun.io/image_advanced/81fc8a255488615510790ee41be314e054978d51/full.png" table Image ( "attr.rust.derive": "PartialEq", "attr.cpp.no_field_ctors", diff --git a/crates/store/re_types/src/archetypes/image.rs b/crates/store/re_types/src/archetypes/image.rs index bcfd540dcbb0..92b6338291f5 100644 --- a/crates/store/re_types/src/archetypes/image.rs +++ b/crates/store/re_types/src/archetypes/image.rs @@ -65,67 +65,65 @@ use ::re_types_core::{DeserializationError, DeserializationResult}; /// /// /// -/// ### Advanced usage of `send_columns` to send multiple images at once +/// ### Logging images with various formats /// ```ignore -/// use ndarray::{s, Array, ShapeBuilder}; -/// use rerun::Archetype as _; +/// use rerun::external::ndarray; /// /// fn main() -> Result<(), Box> { -/// let rec = rerun::RecordingStreamBuilder::new("rerun_example_image_send_columns").spawn()?; +/// let rec = rerun::RecordingStreamBuilder::new("rerun_example_image_formats").spawn()?; /// -/// // Timeline on which the images are distributed. -/// let times = (0..20).collect::>(); +/// // Simple gradient image +/// let image = ndarray::Array3::from_shape_fn((256, 256, 3), |(y, x, c)| match c { +/// 0 => x as u8, +/// 1 => (x + y).min(255) as u8, +/// 2 => y as u8, +/// _ => unreachable!(), +/// }); /// -/// // Create a batch of images with a moving rectangle. -/// let width = 300; -/// let height = 200; -/// let mut images = Array::::zeros((times.len(), height, width, 3).f()) -/// .as_standard_layout() // Make sure the data is laid out as we expect it. -/// .into_owned(); -/// images.slice_mut(s![.., .., .., 2]).fill(255); -/// for &t in × { -/// let t = t as usize; -/// images -/// .slice_mut(s![t, 50..150, (t * 10)..(t * 10 + 100), 1]) -/// .fill(255); -/// } +/// // RGB image +/// rec.log( +/// "image_rgb", +/// &rerun::Image::from_color_model_and_tensor(rerun::ColorModel::RGB, image.clone())?, +/// )?; /// -/// // Log the ImageFormat and indicator once, as static. -/// let format = rerun::components::ImageFormat::rgb8([width as _, height as _]); -/// rec.log_component_batches( -/// "images", -/// true, -/// [&format as _, &rerun::Image::indicator() as _], +/// // Green channel only (Luminance) +/// rec.log( +/// "image_green_only", +/// &rerun::Image::from_color_model_and_tensor( +/// rerun::ColorModel::L, +/// image.slice(ndarray::s![.., .., 1]).to_owned(), +/// )?, /// )?; /// -/// // Split up the image data into several components referencing the underlying data. -/// let image_size_in_bytes = width * height * 3; -/// let blob = rerun::datatypes::Blob::from(images.into_raw_vec_and_offset().0); -/// let image_column = times -/// .iter() -/// .map(|&t| { -/// let byte_offset = image_size_in_bytes * (t as usize); -/// rerun::components::ImageBuffer::from( -/// blob.clone() // Clone is only a reference count increase, not a full copy. -/// .sliced(byte_offset..(byte_offset + image_size_in_bytes)), -/// ) -/// }) -/// .collect::>(); +/// // BGR image +/// rec.log( +/// "image_bgr", +/// &rerun::Image::from_color_model_and_tensor( +/// rerun::ColorModel::BGR, +/// image.slice(ndarray::s![.., .., ..;-1]).to_owned(), +/// )?, +/// )?; /// -/// // Send all images at once. -/// let timeline_values = rerun::TimeColumn::new_sequence("step", times); -/// rec.send_columns("images", [timeline_values], [&image_column as _])?; +/// // New image with Separate Y/U/V planes with 4:2:2 chroma downsampling +/// let mut yuv_bytes = Vec::with_capacity(256 * 256 + 128 * 256 * 2); +/// yuv_bytes.extend(std::iter::repeat(128).take(256 * 256)); // Fixed value for Y. +/// yuv_bytes.extend((0..256).flat_map(|_y| (0..128).map(|x| x * 2))); // Gradient for U. +/// yuv_bytes.extend((0..256).flat_map(|y| std::iter::repeat(y as u8).take(128))); // Gradient for V. +/// rec.log( +/// "image_yuv422", +/// &rerun::Image::from_pixel_format( +/// [256, 256], +/// rerun::PixelFormat::Y_U_V16_FullRange, +/// yuv_bytes, +/// ), +/// )?; /// /// Ok(()) /// } /// ``` ///
/// -/// -/// -/// -/// -/// +/// /// ///
#[derive(Clone, Debug, PartialEq)] diff --git a/docs/content/reference/types/archetypes/image.md b/docs/content/reference/types/archetypes/image.md index a6d23fbe46f8..3f9c8f6a6da1 100644 --- a/docs/content/reference/types/archetypes/image.md +++ b/docs/content/reference/types/archetypes/image.md @@ -48,15 +48,19 @@ snippet: archetypes/image_simple -### Advanced usage of `send_columns` to send multiple images at once +### Logging images with various formats + +snippet: archetypes/image_formats + + + + + +### Image from file, PIL & OpenCV snippet: archetypes/image_send_columns - - - - - + diff --git a/docs/snippets/all/archetypes/image_advanced.py b/docs/snippets/all/archetypes/image_advanced.py index d9959452c041..83c8516a6d45 100644 --- a/docs/snippets/all/archetypes/image_advanced.py +++ b/docs/snippets/all/archetypes/image_advanced.py @@ -25,13 +25,11 @@ image = np.array(PILImage.open(file_path)) rr.log("from_pillow_rgba", rr.Image(image)) -# Convert to RGB, fill transparent pixels with a color, and log the image. +# Drop the alpha channel from the image. image_rgb = image[..., :3] -image_rgb[image[:, :, 3] == 0] = (45, 15, 15) rr.log("from_pillow_rgb", rr.Image(image_rgb)) -# Read with OpenCV +# Read with OpenCV. image = cv2.imread(file_path) - # OpenCV uses BGR ordering, we need to make this known to Rerun. rr.log("from_opencv", rr.Image(image, color_model="BGR")) diff --git a/docs/snippets/all/archetypes/image_formats.cpp b/docs/snippets/all/archetypes/image_formats.cpp new file mode 100644 index 000000000000..077b2bc0ec58 --- /dev/null +++ b/docs/snippets/all/archetypes/image_formats.cpp @@ -0,0 +1,64 @@ +#include +#include +#include + +#include + +int main() { + const auto rec = rerun::RecordingStream("rerun_example_image_formats"); + rec.spawn().exit_on_failure(); + + // Simple gradient image + std::vector image(256 * 256 * 3); + for (size_t y = 0; y < 256; ++y) { + for (size_t x = 0; x < 256; ++x) { + image[(y * 256 + x) * 3 + 0] = static_cast(x); + image[(y * 256 + x) * 3 + 1] = static_cast(std::min(255, x + y)); + image[(y * 256 + x) * 3 + 2] = static_cast(y); + } + } + + // RGB image + rec.log("image_rgb", rerun::Image::from_rgb24(image, {256, 256})); + + // Green channel only (Luminance) + std::vector green_channel(256 * 256); + for (size_t i = 0; i < 256 * 256; ++i) { + green_channel[i] = image[i * 3 + 1]; + } + rec.log( + "image_green_only", + rerun::Image(rerun::borrow(green_channel), {256, 256}, rerun::ColorModel::L) + ); + + // BGR image + std::vector bgr_image(256 * 256 * 3); + for (size_t i = 0; i < 256 * 256; ++i) { + bgr_image[i * 3 + 0] = image[i * 3 + 2]; + bgr_image[i * 3 + 1] = image[i * 3 + 1]; + bgr_image[i * 3 + 2] = image[i * 3 + 0]; + } + rec.log( + "image_bgr", + rerun::Image(rerun::borrow(bgr_image), {256, 256}, rerun::ColorModel::BGR) + ); + + // New image with Separate Y/U/V planes with 4:2:2 chroma downsampling + std::vector yuv_bytes(256 * 256 + 128 * 256 * 2); + std::fill_n(yuv_bytes.begin(), 256 * 256, static_cast(128)); // Fixed value for Y + size_t u_plane_offset = 256 * 256; + size_t v_plane_offset = u_plane_offset + 128 * 256; + for (size_t y = 0; y < 256; ++y) { + for (size_t x = 0; x < 128; ++x) { + auto coord = y * 128 + x; + yuv_bytes[u_plane_offset + coord] = static_cast(x * 2); // Gradient for U + yuv_bytes[v_plane_offset + coord] = static_cast(y); // Gradient for V + } + } + rec.log( + "image_yuv422", + rerun::Image(rerun::borrow(yuv_bytes), {256, 256}, rerun::PixelFormat::Y_U_V16_FullRange) + ); + + return 0; +} diff --git a/docs/snippets/all/archetypes/image_formats.py b/docs/snippets/all/archetypes/image_formats.py new file mode 100644 index 000000000000..7c491e475125 --- /dev/null +++ b/docs/snippets/all/archetypes/image_formats.py @@ -0,0 +1,18 @@ +"""Create and log an image with various formats.""" + +import numpy as np +import rerun as rr + +rr.init("rerun_example_image_formats", spawn=True) + +# Simple gradient image, logged in different formats. +image = np.array([[[x, min(255, x + y), y] for x in range(0, 256)] for y in range(0, 256)], dtype=np.uint8) +rr.log("image_rgb", rr.Image(image)) +rr.log("image_green_only", rr.Image(image[:, :, 1], color_model="l")) # Luminance only +rr.log("image_bgr", rr.Image(image[:, :, ::-1], color_model="bgr")) # BGR + +# New image with Separate Y/U/V planes with 4:2:2 chroma downsampling +y = bytes([128 for y in range(0, 256) for x in range(0, 256)]) +u = bytes([x * 2 for y in range(0, 256) for x in range(0, 128)]) # Half horizontal resolution for chroma. +v = bytes([y for y in range(0, 256) for x in range(0, 128)]) +rr.log("image_yuv422", rr.Image(bytes=y + u + v, width=256, height=256, pixel_format=rr.PixelFormat.Y_U_V16_FullRange)) diff --git a/docs/snippets/all/archetypes/image_formats.rs b/docs/snippets/all/archetypes/image_formats.rs new file mode 100644 index 000000000000..226b14995299 --- /dev/null +++ b/docs/snippets/all/archetypes/image_formats.rs @@ -0,0 +1,53 @@ +use rerun::external::ndarray; + +fn main() -> Result<(), Box> { + let rec = rerun::RecordingStreamBuilder::new("rerun_example_image_formats").spawn()?; + + // Simple gradient image + let image = ndarray::Array3::from_shape_fn((256, 256, 3), |(y, x, c)| match c { + 0 => x as u8, + 1 => (x + y).min(255) as u8, + 2 => y as u8, + _ => unreachable!(), + }); + + // RGB image + rec.log( + "image_rgb", + &rerun::Image::from_color_model_and_tensor(rerun::ColorModel::RGB, image.clone())?, + )?; + + // Green channel only (Luminance) + rec.log( + "image_green_only", + &rerun::Image::from_color_model_and_tensor( + rerun::ColorModel::L, + image.slice(ndarray::s![.., .., 1]).to_owned(), + )?, + )?; + + // BGR image + rec.log( + "image_bgr", + &rerun::Image::from_color_model_and_tensor( + rerun::ColorModel::BGR, + image.slice(ndarray::s![.., .., ..;-1]).to_owned(), + )?, + )?; + + // New image with Separate Y/U/V planes with 4:2:2 chroma downsampling + let mut yuv_bytes = Vec::with_capacity(256 * 256 + 128 * 256 * 2); + yuv_bytes.extend(std::iter::repeat(128).take(256 * 256)); // Fixed value for Y. + yuv_bytes.extend((0..256).flat_map(|_y| (0..128).map(|x| x * 2))); // Gradient for U. + yuv_bytes.extend((0..256).flat_map(|y| std::iter::repeat(y as u8).take(128))); // Gradient for V. + rec.log( + "image_yuv422", + &rerun::Image::from_pixel_format( + [256, 256], + rerun::PixelFormat::Y_U_V16_FullRange, + yuv_bytes, + ), + )?; + + Ok(()) +} diff --git a/rerun_cpp/src/rerun.hpp b/rerun_cpp/src/rerun.hpp index d882f8006697..e15d3ce115d1 100644 --- a/rerun_cpp/src/rerun.hpp +++ b/rerun_cpp/src/rerun.hpp @@ -56,6 +56,7 @@ namespace rerun { using datatypes::Float32; using datatypes::KeypointPair; using datatypes::Mat3x3; + using datatypes::PixelFormat; using datatypes::Quaternion; using datatypes::Rgba32; using datatypes::RotationAxisAngle; diff --git a/rerun_cpp/src/rerun/archetypes/image.hpp b/rerun_cpp/src/rerun/archetypes/image.hpp index a96850434d3e..80d28380825b 100644 --- a/rerun_cpp/src/rerun/archetypes/image.hpp +++ b/rerun_cpp/src/rerun/archetypes/image.hpp @@ -72,57 +72,73 @@ namespace rerun::archetypes { /// } /// ``` /// - /// ### Advanced usage of `send_columns` to send multiple images at once - /// ![image](https://static.rerun.io/image_send_columns/321455161d79e2c45d6f5a6f175d6f765f418897/full.png) + /// ### Logging images with various formats + /// ![image](https://static.rerun.io/image_formats/7b8a162fcfd266f303980439beea997dc8544c24/full.png) /// /// ```cpp - /// #include + /// #include + /// #include + /// #include + /// /// #include /// /// int main() { - /// auto rec = rerun::RecordingStream("rerun_example_image_send_columns"); + /// const auto rec = rerun::RecordingStream("rerun_example_image_formats"); /// rec.spawn().exit_on_failure(); /// - /// // Timeline on which the images are distributed. - /// std::vector times(20); - /// std::iota(times.begin(), times.end(), 0); - /// - /// // Create a batch of images with a moving rectangle. - /// const size_t width = 300, height = 200; - /// std::vector images(times.size() * height * width * 3, 0); - /// for (size_t t = 0; t = 50 && y <150 && x>= t * 10 && x image(256 * 256 * 3); + /// for (size_t y = 0; y <256; ++y) { + /// for (size_t x = 0; x <256; ++x) { + /// image[(y * 256 + x) * 3 + 0] = static_cast(x); + /// image[(y * 256 + x) * 3 + 1] = static_cast(std::min(255, x + y)); + /// image[(y * 256 + x) * 3 + 2] = static_cast(y); /// } /// } /// - /// // Log the ImageFormat and indicator once, as static. - /// auto format = rerun::components::ImageFormat( - /// {width, height}, - /// rerun::ColorModel::RGB, - /// rerun::ChannelDatatype::U8 + /// // RGB image + /// rec.log("image_rgb", rerun::Image::from_rgb24(image, {256, 256})); + /// + /// // Green channel only (Luminance) + /// std::vector green_channel(256 * 256); + /// for (size_t i = 0; i <256 * 256; ++i) { + /// green_channel[i] = image[i * 3 + 1]; + /// } + /// rec.log( + /// "image_green_only", + /// rerun::Image(rerun::borrow(green_channel), {256, 256}, rerun::ColorModel::L) /// ); - /// rec.log_static("images", rerun::borrow(&format), rerun::Image::IndicatorComponent()); /// - /// // Split up the image data into several components referencing the underlying data. - /// const size_t image_size_in_bytes = width * height * 3; - /// std::vector image_data(times.size()); - /// for (size_t i = 0; i bgr_image(256 * 256 * 3); + /// for (size_t i = 0; i <256 * 256; ++i) { + /// bgr_image[i * 3 + 0] = image[i * 3 + 2]; + /// bgr_image[i * 3 + 1] = image[i * 3 + 1]; + /// bgr_image[i * 3 + 2] = image[i * 3 + 0]; /// } + /// rec.log( + /// "image_bgr", + /// rerun::Image(rerun::borrow(bgr_image), {256, 256}, rerun::ColorModel::BGR) + /// ); /// - /// // Send all images at once. - /// rec.send_columns( - /// "images", - /// rerun::TimeColumn::from_sequence_points("step", std::move(times)), - /// rerun::borrow(image_data) + /// // New image with Separate Y/U/V planes with 4:2:2 chroma downsampling + /// std::vector yuv_bytes(256 * 256 + 128 * 256 * 2); + /// std::fill_n(yuv_bytes.begin(), 256 * 256, static_cast(128)); // Fixed value for Y + /// size_t u_plane_offset = 256 * 256; + /// size_t v_plane_offset = u_plane_offset + 128 * 256; + /// for (size_t y = 0; y <256; ++y) { + /// for (size_t x = 0; x <128; ++x) { + /// auto coord = y * 128 + x; + /// yuv_bytes[u_plane_offset + coord] = static_cast(x * 2); // Gradient for U + /// yuv_bytes[v_plane_offset + coord] = static_cast(y); // Gradient for V + /// } + /// } + /// rec.log( + /// "image_yuv422", + /// rerun::Image(rerun::borrow(yuv_bytes), {256, 256}, rerun::PixelFormat::Y_U_V16_FullRange) /// ); + /// + /// return 0; /// } /// ``` struct Image { diff --git a/rerun_py/rerun_sdk/rerun/archetypes/image.py b/rerun_py/rerun_sdk/rerun/archetypes/image.py index 6180b2b98e7c..c0cfb2703d40 100644 --- a/rerun_py/rerun_sdk/rerun/archetypes/image.py +++ b/rerun_py/rerun_sdk/rerun/archetypes/image.py @@ -60,46 +60,28 @@ class Image(ImageExt, Archetype): - ### Advanced usage of `send_columns` to send multiple images at once: + ### Logging images with various formats: ```python import numpy as np import rerun as rr - rr.init("rerun_example_image_send_columns", spawn=True) - - # Timeline on which the images are distributed. - times = np.arange(0, 20) - - # Create a batch of images with a moving rectangle. - width, height = 300, 200 - images = np.zeros((len(times), height, width, 3), dtype=np.uint8) - images[:, :, :, 2] = 255 - for t in times: - images[t, 50:150, (t * 10) : (t * 10 + 100), 1] = 255 - - # Log the ImageFormat and indicator once, as static. - format_static = rr.components.ImageFormat(width=width, height=height, color_model="RGB", channel_datatype="U8") - rr.log("images", [format_static, rr.Image.indicator()], static=True) - - # Send all images at once. - rr.send_columns( - "images", - times=[rr.TimeSequenceColumn("step", times)], - # Reshape the images so `ImageBufferBatch` can tell that this is several blobs. - # - # Note that the `ImageBufferBatch` consumes arrays of bytes, - # so if you have a different channel datatype than `U8`, you need to make sure - # that the data is converted to arrays of bytes before passing it to `ImageBufferBatch`. - components=[rr.components.ImageBufferBatch(images.reshape(len(times), -1))], - ) + rr.init("rerun_example_image_formats", spawn=True) + + # Simple gradient image, logged in different formats. + image = np.array([[[x, min(255, x + y), y] for x in range(0, 256)] for y in range(0, 256)], dtype=np.uint8) + rr.log("image_rgb", rr.Image(image)) + rr.log("image_green_only", rr.Image(image[:, :, 1], color_model="l")) # Luminance only + rr.log("image_bgr", rr.Image(image[:, :, ::-1], color_model="bgr")) # BGR + + # New image with Separate Y/U/V planes with 4:2:2 chroma downsampling + y = bytes([128 for y in range(0, 256) for x in range(0, 256)]) + u = bytes([x * 2 for y in range(0, 256) for x in range(0, 128)]) # Half horizontal resolution for chroma. + v = bytes([y for y in range(0, 256) for x in range(0, 128)]) + rr.log("image_yuv422", rr.Image(bytes=y + u + v, width=256, height=256, pixel_format=rr.PixelFormat.Y_U_V16_FullRange)) ```
- - - - - +