BGR(A) image format support (#7238)

* Fixes #2340

Adds a checklist item that goes through all the different format to make
sure they run properly - this looks a bit overzealous, but in fact we
are using the fact that wgpu has special support for 8bit bgr images in
order to support them on meshes (this detail is tbf untested as of
now!), so going through our formats in a test has quite some merit.
(aaalso I think it's the only place where we actually check now that all
datatypes still work fine 😳 )


![image](https://github.com/user-attachments/assets/49b3b8a3-469d-48ce-a6b6-3205f682d902)
("ethnically sourced lena picture", courtesy of Morten Rieger,
https://mortenhannemose.github.io/lena/)


Speaking of meshes: the mesh texture support story is now spelled out
better, showing warnings and ignoring textures if they're in an
unsupported format.

Author: Wumpf

crates/store/re_types/definitions/rerun/datatypes/color_model.fbs

@@ -15,4 +15,10 @@ enum ColorModel: ubyte{
 
     /// Red, Green, Blue, Alpha
     RGBA = 3,
+
+    /// Blue, Green, Red
+    BGR,
+
+    /// Blue, Green, Red, Alpha
+    BGRA,
 }

crates/store/re_types/src/archetypes/image_ext.rs

@@ -39,10 +39,10 @@ impl Image {
 
         let is_shape_correct = match color_model {
             ColorModel::L => non_empty_dim_inds.len() == 2,
-            ColorModel::RGB => {
+            ColorModel::RGB | ColorModel::BGR => {
                 non_empty_dim_inds.len() == 3 && shape[non_empty_dim_inds[2]].size == 3
             }
-            ColorModel::RGBA => {
+            ColorModel::RGBA | ColorModel::BGRA => {
                 non_empty_dim_inds.len() == 3 && shape[non_empty_dim_inds[2]].size == 4
             }
         };

crates/store/re_types/src/datatypes/color_model.rs

@@ -8,17 +8,17 @@ impl ColorModel {
     pub fn num_channels(self) -> usize {
         match self {
             Self::L => 1,
-            Self::RGB => 3,
-            Self::RGBA => 4,
+            Self::RGB | Self::BGR => 3,
+            Self::RGBA | Self::BGRA => 4,
         }
     }
 
     /// Do we have an alpha channel?
     #[inline]
     pub fn has_alpha(&self) -> bool {
         match self {
-            Self::L | Self::RGB => false,
-            Self::RGBA => true,
+            Self::L | Self::RGB | Self::BGR => false,
+            Self::RGBA | Self::BGRA => true,
         }
     }
 }

crates/store/re_types/src/datatypes/color_model_ext.rs

@@ -421,6 +421,52 @@ fn image_pixel_value_ui(
                     None
                 }
             }
+
+            ColorModel::BGR => {
+                if let Some([b, g, r]) = {
+                    if let [Some(b), Some(g), Some(r)] = [
+                        image.get_xyc(x, y, 0),
+                        image.get_xyc(x, y, 1),
+                        image.get_xyc(x, y, 2),
+                    ] {
+                        Some([r, g, b])
+                    } else {
+                        None
+                    }
+                } {
+                    match (b, g, r) {
+                        (TensorElement::U8(b), TensorElement::U8(g), TensorElement::U8(r)) => {
+                            Some(format!("B: {b}, G: {g}, R: {r}, #{b:02X}{g:02X}{r:02X}"))
+                        }
+                        _ => Some(format!("B: {b}, G: {g}, R: {r}")),
+                    }
+                } else {
+                    None
+                }
+            }
+
+            ColorModel::BGRA => {
+                if let (Some(b), Some(g), Some(r), Some(a)) = (
+                    image.get_xyc(x, y, 0),
+                    image.get_xyc(x, y, 1),
+                    image.get_xyc(x, y, 2),
+                    image.get_xyc(x, y, 3),
+                ) {
+                    match (b, g, r, a) {
+                        (
+                            TensorElement::U8(b),
+                            TensorElement::U8(g),
+                            TensorElement::U8(r),
+                            TensorElement::U8(a),
+                        ) => Some(format!(
+                            "B: {b}, G: {g}, R: {r}, A: {a}, #{r:02X}{g:02X}{b:02X}{a:02X}"
+                        )),
+                        _ => Some(format!("B: {b}, G: {g}, R: {r}, A: {a}")),
+                    }
+                } else {
+                    None
+                }
+            }
         },
     };
 

crates/viewer/re_data_ui/src/image.rs

@@ -59,6 +59,9 @@ struct UniformBuffer {
 
     /// Boolean: multiply RGB with alpha before filtering
     multiply_rgb_with_alpha: u32,
+
+    /// Boolean: swizzle RGBA to BGRA
+    bgra_to_rgba: u32,
 };
 
 @group(1) @binding(0)

crates/viewer/re_renderer/shader/rectangle.wgsl

@@ -28,6 +28,11 @@ fn decode_color(sampled_value: vec4f) -> vec4f {
     // Normalize the value first, otherwise premultiplying alpha and linear space conversion won't make sense.
     var rgba = normalize_range(sampled_value);
 
+    // BGR(A) -> RGB(A)
+    if rect_info.bgra_to_rgba != 0u {
+        rgba = rgba.bgra;
+    }
+
     // Convert to linear space
     if rect_info.decode_srgb != 0u {
         if all(vec3f(0.0) <= rgba.rgb) && all(rgba.rgb <= vec3f(1.0)) {

crates/viewer/re_renderer/shader/rectangle_fs.wgsl

@@ -51,6 +51,10 @@ pub enum TextureFilterMin {
 pub enum ShaderDecoding {
     Nv12,
     Yuy2,
+
+    /// BGR(A)->RGB(A) conversion is done in the shader.
+    /// (as opposed to doing it via ``)
+    Bgr,
 }
 
 /// Describes a texture and how to map it to a color.
@@ -151,7 +155,7 @@ impl ColormappedTexture {
                 let [width, height] = self.texture.width_height();
                 [width / 2, height]
             }
-            _ => self.texture.width_height(),
+            Some(ShaderDecoding::Bgr) | None => self.texture.width_height(),
         }
     }
 }
@@ -275,7 +279,8 @@ mod gpu_data {
 
         decode_srgb: u32,
         multiply_rgb_with_alpha: u32,
-        _row_padding: [u32; 2],
+        bgra_to_rgba: u32,
+        _row_padding: [u32; 1],
 
         _end_padding: [wgpu_buffer_types::PaddingRow; 16 - 7],
     }
@@ -362,6 +367,7 @@ mod gpu_data {
                 super::TextureFilterMag::Linear => FILTER_BILINEAR,
                 super::TextureFilterMag::Nearest => FILTER_NEAREST,
             };
+            let bgra_to_rgba = shader_decoding == &Some(super::ShaderDecoding::Bgr);
 
             Ok(Self {
                 top_left_corner_position: (*top_left_corner_position).into(),
@@ -379,6 +385,7 @@ mod gpu_data {
                 magnification_filter,
                 decode_srgb: *decode_srgb as _,
                 multiply_rgb_with_alpha: *multiply_rgb_with_alpha as _,
+                bgra_to_rgba: bgra_to_rgba as _,
                 _row_padding: Default::default(),
                 _end_padding: Default::default(),
             })

crates/viewer/re_renderer/src/renderer/rectangles.rs

@@ -150,26 +150,19 @@ impl LoadedMesh {
             re_math::BoundingBox::from_points(vertex_positions.iter().copied())
         };
 
-        let albedo = if let (Some(albedo_texture_buffer), Some(albedo_texture_format)) =
-            (&albedo_texture_buffer, albedo_texture_format)
-        {
-            let image_info = ImageInfo {
-                buffer_row_id: RowId::ZERO, // unused
-                buffer: albedo_texture_buffer.0.clone(),
-                format: albedo_texture_format.0,
-                kind: re_types::image::ImageKind::Color,
-                colormap: None,
-            };
-            re_viewer_context::gpu_bridge::get_or_create_texture(render_ctx, texture_key, || {
-                let debug_name = "mesh albedo texture";
-                texture_creation_desc_from_color_image(&image_info, debug_name)
-            })?
-        } else {
+        let albedo = try_get_or_create_albedo_texture(
+            albedo_texture_buffer,
+            albedo_texture_format,
+            render_ctx,
+            texture_key,
+            &name,
+        )
+        .unwrap_or_else(|| {
             render_ctx
                 .texture_manager_2d
                 .white_texture_unorm_handle()
                 .clone()
-        };
+        });
 
         let mesh = re_renderer::mesh::Mesh {
             label: name.clone().into(),
@@ -211,3 +204,44 @@ impl LoadedMesh {
         self.bbox
     }
 }
+
+fn try_get_or_create_albedo_texture(
+    albedo_texture_buffer: &Option<re_types::components::ImageBuffer>,
+    albedo_texture_format: &Option<re_types::components::ImageFormat>,
+    render_ctx: &RenderContext,
+    texture_key: u64,
+    name: &str,
+) -> Option<re_renderer::resource_managers::GpuTexture2D> {
+    let (Some(albedo_texture_buffer), Some(albedo_texture_format)) =
+        (&albedo_texture_buffer, albedo_texture_format)
+    else {
+        return None;
+    };
+
+    let image_info = ImageInfo {
+        buffer_row_id: RowId::ZERO, // unused
+        buffer: albedo_texture_buffer.0.clone(),
+        format: albedo_texture_format.0,
+        kind: re_types::image::ImageKind::Color,
+        colormap: None,
+    };
+
+    if re_viewer_context::gpu_bridge::required_shader_decode(albedo_texture_format).is_some() {
+        re_log::warn_once!("Mesh can't yet handle encoded image formats like NV12 & YUY2 or BGR(A) formats without a channel type other than U8. Ignoring the texture at {name:?}.");
+        return None;
+    }
+
+    let texture =
+        re_viewer_context::gpu_bridge::get_or_create_texture(render_ctx, texture_key, || {
+            let debug_name = "mesh albedo texture";
+            texture_creation_desc_from_color_image(&image_info, debug_name)
+        });
+
+    match texture {
+        Ok(texture) => Some(texture),
+        Err(err) => {
+            re_log::warn_once!("Failed to create mesh albedo texture for {name:?}: {err}");
+            None
+        }
+    }
+}