Expose additional information about decoded frames in the viewer

crates/store/re_video/examples/frames.rs

@@ -84,7 +84,12 @@ fn main() {
                 .truncate(true)
                 .open(output_dir.join(format!("{i:0width$}.ppm")))
                 .expect("failed to open file");
-            write_binary_ppm(&mut file, frame.width, frame.height, &frame.data);
+            write_binary_ppm(
+                &mut file,
+                frame.content.width,
+                frame.content.height,
+                &frame.content.data,
+            );
         }
     }
 }

crates/store/re_video/src/decode/av1.rs

@@ -6,8 +6,8 @@ use crate::Time;
 use dav1d::{PixelLayout, PlanarImageComponent};
 
 use super::{
-    async_decoder_wrapper::SyncDecoder, Chunk, Error, Frame, OutputCallback, PixelFormat, Result,
-    YuvMatrixCoefficients, YuvPixelLayout, YuvRange,
+    async_decoder_wrapper::SyncDecoder, Chunk, Error, Frame, FrameContent, FrameInfo,
+    OutputCallback, PixelFormat, Result, YuvMatrixCoefficients, YuvPixelLayout, YuvRange,
 };
 
 pub struct SyncDav1dDecoder {
@@ -228,12 +228,16 @@ fn output_picture(
     };
 
     let frame = Frame {
-        data,
-        width: picture.width(),
-        height: picture.height(),
-        format,
-        presentation_timestamp: Time(picture.timestamp().unwrap_or(0)),
-        duration: Time(picture.duration()),
+        content: FrameContent {
+            data,
+            width: picture.width(),
+            height: picture.height(),
+            format,
+        },
+        info: FrameInfo {
+            presentation_timestamp: Time(picture.timestamp().unwrap_or(0)),
+            duration: Time(picture.duration()),
+        },
     };
     on_output(Ok(frame));
 }

crates/store/re_video/src/decode/mod.rs

@@ -198,23 +198,49 @@ pub struct Chunk {
     pub duration: Time,
 }
 
+/// Data for a decoded frame on native targets.
 #[cfg(not(target_arch = "wasm32"))]
-pub type FrameData = Vec<u8>;
-
-#[cfg(target_arch = "wasm32")]
-pub type FrameData = webcodecs::WebVideoFrame;
-
-/// One decoded video frame.
-pub struct Frame {
-    pub data: FrameData,
+pub struct FrameContent {
+    pub data: Vec<u8>,
     pub width: u32,
     pub height: u32,
     pub format: PixelFormat,
+}
+
+/// Data for a decoded frame on the web.
+#[cfg(target_arch = "wasm32")]
+pub type FrameContent = webcodecs::WebVideoFrame;
+
+/// Meta information about a decoded video frame, as reported by the decoder.
+#[derive(Debug, Clone)]
+pub struct FrameInfo {
     pub presentation_timestamp: Time,
     pub duration: Time,
 }
 
-/// Pixel format/layout used by [`Frame::data`].
+impl Default for FrameInfo {
+    fn default() -> Self {
+        Self {
+            presentation_timestamp: Time::MAX,
+            duration: Time::MAX,
+        }
+    }
+}
+
+impl FrameInfo {
+    /// Presentation timestamp range in which this frame is valid.
+    pub fn time_range(&self) -> std::ops::Range<Time> {
+        self.presentation_timestamp..self.presentation_timestamp + self.duration
+    }
+}
+
+/// One decoded video frame.
+pub struct Frame {
+    pub content: FrameContent,
+    pub info: FrameInfo,
+}
+
+/// Pixel format/layout used by [`FrameContent::data`].
 #[derive(Debug)]
 pub enum PixelFormat {
     Rgb8Unorm,

crates/store/re_video/src/decode/webcodecs.rs

@@ -8,7 +8,7 @@ use web_sys::{
 };
 
 use super::{
-    AsyncDecoder, Chunk, DecodeHardwareAcceleration, Frame, OutputCallback, PixelFormat, Result,
+    AsyncDecoder, Chunk, DecodeHardwareAcceleration, Frame, FrameInfo, OutputCallback, Result,
 };
 use crate::{Config, Time, Timescale};
 
@@ -179,17 +179,12 @@ fn init_video_decoder(
                 Time::from_micros(frame.timestamp().unwrap_or(0.0), timescale);
             let duration = Time::from_micros(frame.duration().unwrap_or(0.0), timescale);
 
-            let frame = WebVideoFrame(frame);
-            let width = frame.display_width();
-            let height = frame.display_height();
-
             on_output(Ok(Frame {
-                data: frame,
-                width,
-                height,
-                presentation_timestamp,
-                format: PixelFormat::Rgba8Unorm,
-                duration,
+                content: WebVideoFrame(frame),
+                info: FrameInfo {
+                    presentation_timestamp,
+                    duration,
+                },
             }));
         }) as Box<dyn Fn(web_sys::VideoFrame)>)
     };

crates/viewer/re_data_ui/src/blob.rs

@@ -1,6 +1,10 @@
-use re_renderer::{external::re_video::VideoLoadError, video::VideoFrameTexture};
+use re_renderer::{
+    external::re_video::VideoLoadError, resource_managers::SourceImageDataFormat,
+    video::VideoFrameTexture,
+};
 use re_types::components::{Blob, MediaType, VideoTimestamp};
 use re_ui::{list_item::PropertyContent, UiExt};
+use re_video::decode::FrameInfo;
 use re_viewer_context::UiLayout;
 
 use crate::{image::image_preview_ui, EntityDataUi};
@@ -209,7 +213,7 @@ fn show_video_blob_info(
                     // Don't show subsampling mode for monochrome, doesn't make sense usually.
                     if data.is_monochrome() != Some(true) {
                         ui.list_item_flat_noninteractive(
-                            PropertyContent::new("Subsampling mode")
+                            PropertyContent::new("Subsampling")
                                 .value_text(subsampling_mode.to_string()),
                         );
                     }
@@ -272,9 +276,10 @@ fn show_video_blob_info(
                     ) {
                         Ok(VideoFrameTexture {
                             texture,
-                            time_range,
                             is_pending,
                             show_spinner,
+                            frame_info,
+                            source_pixel_format,
                         }) => {
                             let response = crate::image::texture_preview_ui(
                                 render_ctx,
@@ -297,22 +302,12 @@ fn show_video_blob_info(
                                 egui::Spinner::new().paint_at(ui, smaller_rect);
                             }
 
-                            response.on_hover_ui(|ui| {
-                                // Prevent `Area` auto-sizing from shrinking tooltips with dynamic content.
-                                // See https://github.com/emilk/egui/issues/5167
-                                ui.set_max_width(ui.spacing().tooltip_width);
-
-                                let timescale = video.data().timescale;
-                                ui.label(format!(
-                                    "Frame at {} - {}",
-                                    re_format::format_timestamp_seconds(
-                                        time_range.start.into_secs(timescale),
-                                    ),
-                                    re_format::format_timestamp_seconds(
-                                        time_range.end.into_secs(timescale),
-                                    ),
-                                ));
-                            });
+                            decoded_frame_ui(
+                                ui,
+                                &frame_info,
+                                video.data().timescale,
+                                &source_pixel_format,
+                            );
                         }
 
                         Err(err) => {
@@ -341,3 +336,71 @@ fn show_video_blob_info(
         }
     }
 }
+
+fn decoded_frame_ui(
+    ui: &mut egui::Ui,
+    frame_info: &FrameInfo,
+    timescale: re_video::Timescale,
+    source_image_format: &SourceImageDataFormat,
+) {
+    re_ui::list_item::list_item_scope(ui, "decoded_frame_ui", |ui| {
+        let default_open = false;
+        ui.list_item_collapsible_noninteractive_label("Decoded frame info", default_open, |ui| {
+            frame_info_ui(ui, frame_info, timescale);
+            source_image_data_format_ui(ui, source_image_format);
+        });
+    });
+}
+
+fn frame_info_ui(ui: &mut egui::Ui, frame_info: &FrameInfo, timescale: re_video::Timescale) {
+    let time_range = frame_info.time_range();
+    ui.list_item_flat_noninteractive(PropertyContent::new("Time range").value_text(format!(
+        "{} - {}",
+        re_format::format_timestamp_seconds(time_range.start.into_secs(timescale),),
+        re_format::format_timestamp_seconds(time_range.end.into_secs(timescale),),
+    )))
+    .on_hover_text("Time range in which this frame is valid.");
+
+    ui.list_item_flat_noninteractive(
+        PropertyContent::new("PTS").value_text(format!("{}", frame_info.presentation_timestamp.0)),
+    )
+    .on_hover_text("Raw presentation timestamp prior to applying the timescale.
+This specifies the time at which the frame should be shown relative to the start of a video stream.");
+}
+
+fn source_image_data_format_ui(ui: &mut egui::Ui, format: &SourceImageDataFormat) {
+    let label = "Output format";
+
+    match format {
+        SourceImageDataFormat::WgpuCompatible(format) => {
+            ui.list_item_flat_noninteractive(PropertyContent::new(label).value_text(format!("{format:?}")))
+                // This is true for YUV outputs as well, but for RGB/RGBA there was almost certainly some postprocessing involved,
+                // whereas it would very surprising for YUV.
+                .on_hover_text("Pixel format as returned from the decoder.
+Decoders may do arbitrary post processing, so this is not necessarily the format that is actually encoded in the video data!"
+            );
+        }
+
+        SourceImageDataFormat::Yuv {
+            layout,
+            range,
+            coefficients,
+        } => {
+            let default_open = true;
+            ui.list_item_collapsible_noninteractive_label(label, default_open, |ui| {
+                ui.list_item_flat_noninteractive(
+                    PropertyContent::new("Data layout").value_text(layout.to_string()),
+                )
+                .on_hover_text("Subsampling ratio & layout of the pixel data.");
+                ui.list_item_flat_noninteractive(
+                    PropertyContent::new("Color range").value_text(range.to_string()),
+                )
+                .on_hover_text("Valid range of the pixel data values.");
+                ui.list_item_flat_noninteractive(
+                    PropertyContent::new("Yuv Coefficients").value_text(coefficients.to_string()),
+                )
+                .on_hover_text("Matrix coefficients used to convert the pixel data to RGB.");
+            });
+        }
+    };
+}

crates/viewer/re_renderer/src/resource_managers/yuv_converter.rs

@@ -172,6 +172,19 @@ pub enum YuvPixelLayout {
     Y400 = 300,
 }
 
+impl std::fmt::Display for YuvPixelLayout {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Y_U_V444 => write!(f, "4:4:4 (planar)"),
+            Self::Y_U_V422 => write!(f, "4:2:2 (planar)"),
+            Self::Y_U_V420 => write!(f, "4:2:0 (planar)"),
+            Self::Y_UV420 => write!(f, "4:2:0 (semi-planar)"),
+            Self::YUYV422 => write!(f, "4:2:2 (interleaved"),
+            Self::Y400 => write!(f, "4:0:0"),
+        }
+    }
+}
+
 /// Yuv matrix coefficients that determine how a YUV image is meant to be converted to RGB.
 ///
 /// A rigorious definition of the yuv conversion matrix would still require to define
@@ -217,6 +230,16 @@ pub enum YuvMatrixCoefficients {
     // BT2020_NonConstantLuminance,
 }
 
+impl std::fmt::Display for YuvMatrixCoefficients {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Identity => write!(f, "identity"),
+            Self::Bt601 => write!(f, "BT.601"),
+            Self::Bt709 => write!(f, "BT.709"),
+        }
+    }
+}
+
 /// Expected range of YUV values.
 ///
 /// Keep indices in sync with `yuv_converter.wgsl`
@@ -235,6 +258,15 @@ pub enum YuvRange {
     Full = 1,
 }
 
+impl std::fmt::Display for YuvRange {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Limited => write!(f, "limited"),
+            Self::Full => write!(f, "full"),
+        }
+    }
+}
+
 impl YuvPixelLayout {
     /// Given the dimensions of the output picture, what are the expected dimensions of the input data texture.
     pub fn data_texture_width_height(&self, [decoded_width, decoded_height]: [u32; 2]) -> [u32; 2] {