Improve video information on select, warn on HDR content (#7633)

### What
* Uses rerun-io/re_mp4#10

Improves the video blob info in two ways:
* Adds bit depth (if known)
* Shows human-redable name of codec (e.g. AV1 or H.264)

![Screenshot 2024-10-08 at 15 25
24](https://github.com/user-attachments/assets/8a104b85-32fa-4176-8fa3-bd20bca01df6)

![Screenshot 2024-10-08 at 15 25
35](https://github.com/user-attachments/assets/db377cb7-d960-4edc-ad6b-ad60a3363e12)

![Screenshot 2024-10-08 at 15 27
41](https://github.com/user-attachments/assets/cebcb06b-7e16-4101-b9c1-dee549b6038e)


A warning will also be printed if trying to decode a HDR video (since we
don't yet support that).

Related:
* #7594

### Checklist
* [x] I have read and agree to [Contributor
Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and
the [Code of
Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md)
* [x] I've included a screenshot or gif (if applicable)
* [x] I have tested the web demo (if applicable):
* Using examples from latest `main` build:
[rerun.io/viewer](https://rerun.io/viewer/pr/7633?manifest_url=https://app.rerun.io/version/main/examples_manifest.json)
* Using full set of examples from `nightly` build:
[rerun.io/viewer](https://rerun.io/viewer/pr/7633?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json)
* [x] The PR title and labels are set such as to maximize their
usefulness for the next release's CHANGELOG
* [x] If applicable, add a new check to the [release
checklist](https://github.com/rerun-io/rerun/blob/main/tests/python/release_checklist)!
* [x] If have noted any breaking changes to the log API in
`CHANGELOG.md` and the migration guide

- [PR Build Summary](https://build.rerun.io/pr/7633)
- [Recent benchmark results](https://build.rerun.io/graphs/crates.html)
- [Wasm size tracking](https://build.rerun.io/graphs/sizes.html)

To run all checks from `main`, comment on the PR with `@rerun-bot
full-check`.

Cargo.lock

@@ -5367,7 +5367,7 @@ dependencies = [
 [[package]]
 name = "re_mp4"
 version = "0.1.0"
-source = "git+https://github.com/rerun-io/re_mp4?rev=4705e85f62ddb47c32d9c091d8f0662068211bc8#4705e85f62ddb47c32d9c091d8f0662068211bc8"
+source = "git+https://github.com/rerun-io/re_mp4?rev=8614aae24a7a39a5e7de615d83dbcbfadde7f606#8614aae24a7a39a5e7de615d83dbcbfadde7f606"
 dependencies = [
  "byteorder",
  "bytes",

Cargo.toml

@@ -551,9 +551,8 @@ missing_errors_doc = "allow"
 
 # egui_commonmark = { git = "https://github.com/rerun-io/egui_commonmark", rev = "7a9dc755bfa351a3796274cb8ca87129b051c084" } # https://github.com/lampsitter/egui_commonmark/pull/65
 
-# commit on `rerun-io/mp4` `master` branch: https://github.com/rerun-io/re_mp4/tree/master
-# https://github.com/rerun-io/mp4/commit/3236c76f9228cf6ab0b2bfb1b8f9ffcde975ea05
-re_mp4 = { git = "https://github.com/rerun-io/re_mp4", rev = "4705e85f62ddb47c32d9c091d8f0662068211bc8" }
+re_mp4 = { git = "https://github.com/rerun-io/re_mp4", rev = "8614aae24a7a39a5e7de615d83dbcbfadde7f606" } # main 2024-10-08
+# re_mp4 = { path = "../re_mp4" }
 
 # commit on `rerun-io/re_arrow2` `main` branch
 # https://github.com/rerun-io/re_arrow2/commit/e4717d6debc6d4474ec10db8f629f823f57bad07

crates/store/re_log_types/src/time.rs

@@ -535,6 +535,13 @@ impl Duration {
     }
 }
 
+impl From<std::time::Duration> for Duration {
+    #[inline]
+    fn from(duration: std::time::Duration) -> Self {
+        Self::from_nanos(duration.as_nanos() as _)
+    }
+}
+
 impl std::ops::Neg for Duration {
     type Output = Self;
 

crates/store/re_video/src/demux/mod.rs

@@ -70,16 +70,10 @@ impl VideoData {
         }
     }
 
-    /// Duration of the video, in seconds.
+    /// Length of the video.
     #[inline]
-    pub fn duration_sec(&self) -> f64 {
-        self.duration.into_secs(self.timescale)
-    }
-
-    /// Duration of the video, in milliseconds.
-    #[inline]
-    pub fn duration_ms(&self) -> f64 {
-        self.duration.into_millis(self.timescale)
+    pub fn duration(&self) -> std::time::Duration {
+        std::time::Duration::from_nanos(self.duration.into_nanos(self.timescale) as _)
     }
 
     /// Natural width of the video.
@@ -96,8 +90,24 @@ impl VideoData {
 
     /// The codec used to encode the video.
     #[inline]
-    pub fn codec(&self) -> &str {
-        &self.config.codec
+    pub fn human_readable_codec_string(&self) -> String {
+        let human_readable = match &self.config.stsd.contents {
+            re_mp4::StsdBoxContent::Av01(_) => "AV1",
+            re_mp4::StsdBoxContent::Avc1(_) => "H.264",
+            re_mp4::StsdBoxContent::Hvc1(_) => "H.265 HVC1",
+            re_mp4::StsdBoxContent::Hev1(_) => "H.265 HEV1",
+            re_mp4::StsdBoxContent::Vp08(_) => "VP8",
+            re_mp4::StsdBoxContent::Vp09(_) => "VP9",
+            re_mp4::StsdBoxContent::Mp4a(_) => "AAC",
+            re_mp4::StsdBoxContent::Tx3g(_) => "TTXT",
+            re_mp4::StsdBoxContent::Unknown(_) => "Unknown",
+        };
+
+        if let Some(codec) = self.config.stsd.contents.codec_string() {
+            format!("{human_readable} ({codec})")
+        } else {
+            human_readable.to_owned()
+        }
     }
 
     /// The number of samples in the video.
@@ -191,10 +201,8 @@ pub struct Sample {
 /// Configuration of a video.
 #[derive(Debug, Clone)]
 pub struct Config {
-    /// String used to identify the codec and some of its configuration.
-    ///
-    /// e.g. "av01.0.05M.08" (AV1)
-    pub codec: String,
+    /// Contains info about the codec, bit depth, etc.
+    pub stsd: re_mp4::StsdBox,
 
     /// Codec-specific configuration.
     pub description: Vec<u8>,
@@ -208,7 +216,7 @@ pub struct Config {
 
 impl Config {
     pub fn is_av1(&self) -> bool {
-        self.codec.starts_with("av01")
+        matches!(self.stsd.contents, re_mp4::StsdBoxContent::Av01 { .. })
     }
 }
 

crates/store/re_video/src/demux/mp4.rs

@@ -16,9 +16,8 @@ impl VideoData {
             .find(|t| t.kind == Some(re_mp4::TrackKind::Video))
             .ok_or_else(|| VideoLoadError::NoVideoTrack)?;
 
-        let codec = track
-            .codec_string(&mp4)
-            .ok_or_else(|| VideoLoadError::UnsupportedCodec(unknown_codec_fourcc(&mp4, track)))?;
+        let stsd = track.trak(&mp4).mdia.minf.stbl.stsd.clone();
+
         let description = track
             .raw_codec_config(&mp4)
             .ok_or_else(|| VideoLoadError::UnsupportedCodec(unknown_codec_fourcc(&mp4, track)))?;
@@ -27,7 +26,7 @@ impl VideoData {
         let coded_width = track.width;
 
         let config = Config {
-            codec,
+            stsd,
             description,
             coded_height,
             coded_width,

crates/store/re_video/src/lib.rs

@@ -67,6 +67,12 @@ impl Time {
         Self::from_secs(v as f64 / 1e9, timescale)
     }
 
+    /// Convert to a duration
+    #[inline]
+    pub fn duration(self, timescale: Timescale) -> std::time::Duration {
+        std::time::Duration::from_nanos(self.into_nanos(timescale) as _)
+    }
+
     #[inline]
     pub fn into_secs(self, timescale: Timescale) -> f64 {
         self.0 as f64 / timescale.0 as f64

crates/viewer/re_data_ui/src/blob.rs

@@ -193,22 +193,25 @@ fn show_video_blob_info(
                         data.height()
                     )),
                 );
-                ui.list_item_flat_noninteractive(PropertyContent::new("Duration").value_text(
-                    format!(
-                        "{}",
-                        re_log_types::Duration::from_millis(data.duration_ms() as i64)
-                    ),
-                ));
+                if let Some(bit_depth) = data.config.stsd.contents.bit_depth() {
+                    ui.list_item_flat_noninteractive(
+                        PropertyContent::new("Bit depth").value_text(bit_depth.to_string()),
+                    );
+                }
+                ui.list_item_flat_noninteractive(
+                    PropertyContent::new("Duration")
+                        .value_text(format!("{}", re_log_types::Duration::from(data.duration()))),
+                );
                 // Some people may think that num_frames / duration = fps, but that's not true, videos may have variable frame rate.
                 // At the same time, we don't want to overload users with video codec/container specific stuff that they have to understand,
                 // and for all intents and purposes one sample = one frame.
                 // So the compromise is that we truthfully show the number of *samples* here and don't talk about frames.
                 ui.list_item_flat_noninteractive(
                     PropertyContent::new("Sample count")
-                        .value_text(format!("{}", data.num_samples())),
+                        .value_text(re_format::format_uint(data.num_samples())),
                 );
                 ui.list_item_flat_noninteractive(
-                    PropertyContent::new("Codec").value_text(data.codec()),
+                    PropertyContent::new("Codec").value_text(data.human_readable_codec_string()),
                 );
 
                 if ui_layout != UiLayout::Tooltip {
@@ -238,7 +241,7 @@ fn show_video_blob_info(
                         // but the point here is not to have a nice viewer,
                         // but to show the user what they have selected
                         ui.ctx().request_repaint(); // TODO(emilk): schedule a repaint just in time for the next frame of video
-                        ui.input(|i| i.time) % video.data().duration_sec()
+                        ui.input(|i| i.time) % video.data().duration().as_secs_f64()
                     };
 
                     let decode_stream_id = re_renderer::video::VideoDecodingStreamId(

crates/viewer/re_renderer/src/video/decoder/mod.rs

@@ -110,7 +110,19 @@ impl VideoDecoder {
             unused
         )]
 
-        let debug_name = format!("{debug_name}, codec: {}", data.config.codec);
+        let debug_name = format!(
+            "{debug_name}, codec: {}",
+            data.human_readable_codec_string()
+        );
+
+        if let Some(bit_depth) = data.config.stsd.contents.bit_depth() {
+            #[allow(clippy::comparison_chain)]
+            if bit_depth < 8 {
+                re_log::warn_once!("{debug_name} has unusual bit_depth of {bit_depth}");
+            } else if 8 < bit_depth {
+                re_log::warn_once!("{debug_name}: HDR videos not supported. See https://github.com/rerun-io/rerun/issues/7594 for more.");
+            }
+        }
 
         cfg_if::cfg_if! {
             if #[cfg(target_arch = "wasm32")] {
@@ -119,7 +131,7 @@ impl VideoDecoder {
             } else if #[cfg(feature = "video_av1")] {
                 if !data.config.is_av1() {
                     return Err(DecodingError::UnsupportedCodec {
-                        codec: data.config.codec.clone(),
+                        codec: data.human_readable_codec_string(),
                     });
                 }
 

crates/viewer/re_renderer/src/video/decoder/web.rs

@@ -326,7 +326,7 @@ fn js_video_decoder_config(
     config: &re_video::Config,
     hw_acceleration: DecodeHardwareAcceleration,
 ) -> VideoDecoderConfig {
-    let js = VideoDecoderConfig::new(&config.codec);
+    let js = VideoDecoderConfig::new(&config.stsd.contents.codec_string().unwrap_or_default());
     js.set_coded_width(config.coded_width as u32);
     js.set_coded_height(config.coded_height as u32);
     let description = Uint8Array::new_with_length(config.description.len() as u32);