diff --git a/Cargo.lock b/Cargo.lock index 515f4766969d..f6229dfbdfac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5367,7 +5367,7 @@ dependencies = [ [[package]] name = "re_mp4" version = "0.1.0" -source = "git+https://github.com/rerun-io/re_mp4?rev=4705e85f62ddb47c32d9c091d8f0662068211bc8#4705e85f62ddb47c32d9c091d8f0662068211bc8" +source = "git+https://github.com/rerun-io/re_mp4?rev=8614aae24a7a39a5e7de615d83dbcbfadde7f606#8614aae24a7a39a5e7de615d83dbcbfadde7f606" dependencies = [ "byteorder", "bytes", diff --git a/Cargo.toml b/Cargo.toml index dd4eb5e0059c..bfc5adcfa9f6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -551,9 +551,8 @@ missing_errors_doc = "allow" # egui_commonmark = { git = "https://github.com/rerun-io/egui_commonmark", rev = "7a9dc755bfa351a3796274cb8ca87129b051c084" } # https://github.com/lampsitter/egui_commonmark/pull/65 -# commit on `rerun-io/mp4` `master` branch: https://github.com/rerun-io/re_mp4/tree/master -# https://github.com/rerun-io/mp4/commit/3236c76f9228cf6ab0b2bfb1b8f9ffcde975ea05 -re_mp4 = { git = "https://github.com/rerun-io/re_mp4", rev = "4705e85f62ddb47c32d9c091d8f0662068211bc8" } +re_mp4 = { git = "https://github.com/rerun-io/re_mp4", rev = "8614aae24a7a39a5e7de615d83dbcbfadde7f606" } # main 2024-10-08 +# re_mp4 = { path = "../re_mp4" } # commit on `rerun-io/re_arrow2` `main` branch # https://github.com/rerun-io/re_arrow2/commit/e4717d6debc6d4474ec10db8f629f823f57bad07 diff --git a/crates/store/re_log_types/src/time.rs b/crates/store/re_log_types/src/time.rs index 36cf68be284f..a00cb37144e6 100644 --- a/crates/store/re_log_types/src/time.rs +++ b/crates/store/re_log_types/src/time.rs @@ -535,6 +535,13 @@ impl Duration { } } +impl From for Duration { + #[inline] + fn from(duration: std::time::Duration) -> Self { + Self::from_nanos(duration.as_nanos() as _) + } +} + impl std::ops::Neg for Duration { type Output = Self; diff --git a/crates/store/re_video/src/demux/mod.rs b/crates/store/re_video/src/demux/mod.rs index 8b70c440e0bb..f2b495f087e2 100644 --- a/crates/store/re_video/src/demux/mod.rs +++ b/crates/store/re_video/src/demux/mod.rs @@ -70,16 +70,10 @@ impl VideoData { } } - /// Duration of the video, in seconds. + /// Length of the video. #[inline] - pub fn duration_sec(&self) -> f64 { - self.duration.into_secs(self.timescale) - } - - /// Duration of the video, in milliseconds. - #[inline] - pub fn duration_ms(&self) -> f64 { - self.duration.into_millis(self.timescale) + pub fn duration(&self) -> std::time::Duration { + std::time::Duration::from_nanos(self.duration.into_nanos(self.timescale) as _) } /// Natural width of the video. @@ -96,8 +90,24 @@ impl VideoData { /// The codec used to encode the video. #[inline] - pub fn codec(&self) -> &str { - &self.config.codec + pub fn human_readable_codec_string(&self) -> String { + let human_readable = match &self.config.stsd.contents { + re_mp4::StsdBoxContent::Av01(_) => "AV1", + re_mp4::StsdBoxContent::Avc1(_) => "H.264", + re_mp4::StsdBoxContent::Hvc1(_) => "H.265 HVC1", + re_mp4::StsdBoxContent::Hev1(_) => "H.265 HEV1", + re_mp4::StsdBoxContent::Vp08(_) => "VP8", + re_mp4::StsdBoxContent::Vp09(_) => "VP9", + re_mp4::StsdBoxContent::Mp4a(_) => "AAC", + re_mp4::StsdBoxContent::Tx3g(_) => "TTXT", + re_mp4::StsdBoxContent::Unknown(_) => "Unknown", + }; + + if let Some(codec) = self.config.stsd.contents.codec_string() { + format!("{human_readable} ({codec})") + } else { + human_readable.to_owned() + } } /// The number of samples in the video. @@ -191,10 +201,8 @@ pub struct Sample { /// Configuration of a video. #[derive(Debug, Clone)] pub struct Config { - /// String used to identify the codec and some of its configuration. - /// - /// e.g. "av01.0.05M.08" (AV1) - pub codec: String, + /// Contains info about the codec, bit depth, etc. + pub stsd: re_mp4::StsdBox, /// Codec-specific configuration. pub description: Vec, @@ -208,7 +216,7 @@ pub struct Config { impl Config { pub fn is_av1(&self) -> bool { - self.codec.starts_with("av01") + matches!(self.stsd.contents, re_mp4::StsdBoxContent::Av01 { .. }) } } diff --git a/crates/store/re_video/src/demux/mp4.rs b/crates/store/re_video/src/demux/mp4.rs index 14732f1f0469..24659f078cb9 100644 --- a/crates/store/re_video/src/demux/mp4.rs +++ b/crates/store/re_video/src/demux/mp4.rs @@ -16,9 +16,8 @@ impl VideoData { .find(|t| t.kind == Some(re_mp4::TrackKind::Video)) .ok_or_else(|| VideoLoadError::NoVideoTrack)?; - let codec = track - .codec_string(&mp4) - .ok_or_else(|| VideoLoadError::UnsupportedCodec(unknown_codec_fourcc(&mp4, track)))?; + let stsd = track.trak(&mp4).mdia.minf.stbl.stsd.clone(); + let description = track .raw_codec_config(&mp4) .ok_or_else(|| VideoLoadError::UnsupportedCodec(unknown_codec_fourcc(&mp4, track)))?; @@ -27,7 +26,7 @@ impl VideoData { let coded_width = track.width; let config = Config { - codec, + stsd, description, coded_height, coded_width, diff --git a/crates/store/re_video/src/lib.rs b/crates/store/re_video/src/lib.rs index dbde69d6d1b1..106bc84f0222 100644 --- a/crates/store/re_video/src/lib.rs +++ b/crates/store/re_video/src/lib.rs @@ -67,6 +67,12 @@ impl Time { Self::from_secs(v as f64 / 1e9, timescale) } + /// Convert to a duration + #[inline] + pub fn duration(self, timescale: Timescale) -> std::time::Duration { + std::time::Duration::from_nanos(self.into_nanos(timescale) as _) + } + #[inline] pub fn into_secs(self, timescale: Timescale) -> f64 { self.0 as f64 / timescale.0 as f64 diff --git a/crates/viewer/re_data_ui/src/blob.rs b/crates/viewer/re_data_ui/src/blob.rs index 719b152849b4..805513a7c5c2 100644 --- a/crates/viewer/re_data_ui/src/blob.rs +++ b/crates/viewer/re_data_ui/src/blob.rs @@ -193,22 +193,25 @@ fn show_video_blob_info( data.height() )), ); - ui.list_item_flat_noninteractive(PropertyContent::new("Duration").value_text( - format!( - "{}", - re_log_types::Duration::from_millis(data.duration_ms() as i64) - ), - )); + if let Some(bit_depth) = data.config.stsd.contents.bit_depth() { + ui.list_item_flat_noninteractive( + PropertyContent::new("Bit depth").value_text(bit_depth.to_string()), + ); + } + ui.list_item_flat_noninteractive( + PropertyContent::new("Duration") + .value_text(format!("{}", re_log_types::Duration::from(data.duration()))), + ); // Some people may think that num_frames / duration = fps, but that's not true, videos may have variable frame rate. // At the same time, we don't want to overload users with video codec/container specific stuff that they have to understand, // and for all intents and purposes one sample = one frame. // So the compromise is that we truthfully show the number of *samples* here and don't talk about frames. ui.list_item_flat_noninteractive( PropertyContent::new("Sample count") - .value_text(format!("{}", data.num_samples())), + .value_text(re_format::format_uint(data.num_samples())), ); ui.list_item_flat_noninteractive( - PropertyContent::new("Codec").value_text(data.codec()), + PropertyContent::new("Codec").value_text(data.human_readable_codec_string()), ); if ui_layout != UiLayout::Tooltip { @@ -238,7 +241,7 @@ fn show_video_blob_info( // but the point here is not to have a nice viewer, // but to show the user what they have selected ui.ctx().request_repaint(); // TODO(emilk): schedule a repaint just in time for the next frame of video - ui.input(|i| i.time) % video.data().duration_sec() + ui.input(|i| i.time) % video.data().duration().as_secs_f64() }; let decode_stream_id = re_renderer::video::VideoDecodingStreamId( diff --git a/crates/viewer/re_renderer/src/video/decoder/mod.rs b/crates/viewer/re_renderer/src/video/decoder/mod.rs index 1ce8d1363c89..06270b5c68f4 100644 --- a/crates/viewer/re_renderer/src/video/decoder/mod.rs +++ b/crates/viewer/re_renderer/src/video/decoder/mod.rs @@ -110,7 +110,19 @@ impl VideoDecoder { unused )] - let debug_name = format!("{debug_name}, codec: {}", data.config.codec); + let debug_name = format!( + "{debug_name}, codec: {}", + data.human_readable_codec_string() + ); + + if let Some(bit_depth) = data.config.stsd.contents.bit_depth() { + #[allow(clippy::comparison_chain)] + if bit_depth < 8 { + re_log::warn_once!("{debug_name} has unusual bit_depth of {bit_depth}"); + } else if 8 < bit_depth { + re_log::warn_once!("{debug_name}: HDR videos not supported. See https://github.com/rerun-io/rerun/issues/7594 for more."); + } + } cfg_if::cfg_if! { if #[cfg(target_arch = "wasm32")] { @@ -119,7 +131,7 @@ impl VideoDecoder { } else if #[cfg(feature = "video_av1")] { if !data.config.is_av1() { return Err(DecodingError::UnsupportedCodec { - codec: data.config.codec.clone(), + codec: data.human_readable_codec_string(), }); } diff --git a/crates/viewer/re_renderer/src/video/decoder/web.rs b/crates/viewer/re_renderer/src/video/decoder/web.rs index 3562eb68fc97..06a1bb8c62e8 100644 --- a/crates/viewer/re_renderer/src/video/decoder/web.rs +++ b/crates/viewer/re_renderer/src/video/decoder/web.rs @@ -326,7 +326,7 @@ fn js_video_decoder_config( config: &re_video::Config, hw_acceleration: DecodeHardwareAcceleration, ) -> VideoDecoderConfig { - let js = VideoDecoderConfig::new(&config.codec); + let js = VideoDecoderConfig::new(&config.stsd.contents.codec_string().unwrap_or_default()); js.set_coded_width(config.coded_width as u32); js.set_coded_height(config.coded_height as u32); let description = Uint8Array::new_with_length(config.description.len() as u32);