Merge branch 'main' into antoine/snippet-mypy

Author: abey79

.github/workflows/contrib_checks.yml

@@ -123,6 +123,12 @@ jobs:
       - name: Setup software rasterizer
         run: pixi run python ./scripts/ci/setup_software_rasterizer.py
 
+      # Recommended way to install nextest on CI.
+      - name: Install latest nextest release
+        uses: taiki-e/[email protected]
+        with:
+          tool: [email protected]
+
       - name: Rust checks & tests
         run: pixi run rs-check --skip individual_crates docs_slow
 

Cargo.lock

@@ -3742,10 +3742,12 @@ checksum = "5c163c633eb184a4ad2a5e7a5dacf12a58c830d717a7963563d4eceb4ced079f"
 dependencies = [
  "jiff-static",
  "jiff-tzdb-platform",
+ "js-sys",
  "log",
  "portable-atomic",
  "portable-atomic-util",
  "serde",
+ "wasm-bindgen",
  "windows-sys 0.59.0",
 ]
 
@@ -5777,6 +5779,7 @@ name = "re_arrow_util"
 version = "0.23.0-alpha.1+dev"
 dependencies = [
  "arrow",
+ "insta",
  "itertools 0.13.0",
  "re_log",
  "re_tracing",
@@ -6173,7 +6176,6 @@ dependencies = [
  "re_query",
  "re_smart_channel",
  "re_tracing",
- "re_types",
  "re_types_core",
  "serde",
  "similar-asserts",

Cargo.toml

@@ -212,7 +212,7 @@ indicatif = "0.17.7" # Progress bar
 infer = "0.16" # infer MIME type by checking the magic number signaturefer MIME type by checking the magic number signature
 insta = "1.23"
 itertools = "0.13"
-jiff = "0.2.3"
+jiff = { version = "0.2.3", features = ["js"] }
 js-sys = "0.3"
 jsonwebtoken = { version = "9", default-features = false }
 libc = "0.2"

crates/build/re_types_builder/src/codegen/python/mod.rs

@@ -2654,8 +2654,15 @@ fn quote_columnar_methods(reporter: &Reporter, obj: &Object, objects: &Objects)
                     param = kwargs[batch.component_descriptor().archetype_field_name] # type: ignore[index]
                     shape = np.shape(param)  # type: ignore[arg-type]
 
-                    batch_length = shape[1] if len(shape) > 1 else 1 # type: ignore[redundant-expr,misc]
-                    num_rows = shape[0] if len(shape) >= 1 else 1    # type: ignore[redundant-expr,misc]
+                    if pa.types.is_fixed_size_list(arrow_array.type) and len(shape) <= 2:
+                        # If shape length is 2 or less, we have `num_rows` single element batches (each element is a fixed sized list).
+                        # `shape[1]` should be the length of the fixed sized list.
+                        # (This should have been already validated by conversion to the arrow_array)
+                        batch_length = 1
+                    else:
+                        batch_length = shape[1] if len(shape) > 1 else 1  # type: ignore[redundant-expr,misc]
+
+                    num_rows = shape[0] if len(shape) >= 1 else 1  # type: ignore[redundant-expr,misc]
                     sizes = batch_length * np.ones(num_rows)
                 else:
                     # For non-primitive types, default to partitioning each element separately.

crates/store/re_data_loader/src/lerobot.rs

@@ -143,7 +143,7 @@ pub enum LeRobotError {
 ///
 /// Each episode is identified by a unique index and mapped to its corresponding chunk, based on the number of episodes
 /// per chunk (which can be found in `meta/info.json`).
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct LeRobotDataset {
     pub path: PathBuf,
     pub metadata: LeRobotDatasetMetadata,
@@ -216,7 +216,7 @@ impl LeRobotDataset {
 ///
 /// This is a wrapper struct for the metadata files in the `meta` directory of a
 /// `LeRobot` dataset. For more see [`LeRobotDataset`].
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 #[allow(dead_code)] // TODO(gijsd): The list of tasks is not used yet!
 pub struct LeRobotDatasetMetadata {
     pub info: LeRobotDatasetInfo,
@@ -251,7 +251,7 @@ impl LeRobotDatasetMetadata {
 ///
 /// This struct contains the metadata for a `LeRobot` dataset, and is loaded from the `meta/info.json` file
 /// of the dataset.
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct LeRobotDatasetInfo {
     /// The type of the robot.
     pub robot_type: String,
@@ -381,7 +381,7 @@ impl LeRobotDatasetInfo {
 ///
 /// For example, a shape of `[3, 224, 224]` for a [`DType::Image`] feature denotes a 3-channel (e.g. RGB)
 /// image with a height and width of 224 pixels each.
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct Feature {
     pub dtype: DType,
     pub shape: Vec<usize>,
@@ -407,7 +407,7 @@ pub enum DType {
 /// - A flat list of names for each dimension of a feature (e.g., `["height", "width", "channel"]`).
 /// - A nested list of names for each dimension of a feature (e.g., `[[""kLeftShoulderPitch", "kLeftShoulderRoll"]]`)
 /// - A list specific to motors (e.g., `{ "motors": ["motor_0", "motor_1", ...] }`).
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Serialize, Deserialize, Clone)]
 #[serde(untagged)]
 pub enum Names {
     Motors { motors: Vec<String> },
@@ -426,7 +426,7 @@ impl Names {
 
 /// A wrapper struct that deserializes flat or nested lists of strings
 /// into a single flattened [`Vec`] of names for easy indexing.
-#[derive(Debug, Serialize)]
+#[derive(Debug, Serialize, Clone)]
 pub struct NamesList(Vec<String>);
 
 impl<'de> Deserialize<'de> for NamesList {
@@ -482,7 +482,7 @@ pub struct EpisodeIndex(pub usize);
 /// An episode in a `LeRobot` dataset.
 ///
 /// Each episode contains its index, a list of associated tasks, and its total length in frames.
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct LeRobotDatasetEpisode {
     #[serde(rename = "episode_index")]
     pub index: EpisodeIndex,
@@ -498,7 +498,7 @@ pub struct TaskIndex(pub usize);
 /// A task in a `LeRobot` dataset.
 ///
 /// Each task consists of its index and a task description.
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct LeRobotDatasetTask {
     #[serde(rename = "task_index")]
     pub index: TaskIndex,

crates/store/re_entity_db/Cargo.toml

@@ -53,7 +53,6 @@ web-time.workspace = true
 
 [dev-dependencies]
 re_log_encoding = { workspace = true, features = ["decoder", "encoder"] }
-re_types.workspace = true
 
 anyhow.workspace = true
 criterion.workspace = true

crates/store/re_entity_db/tests/clear.rs

@@ -10,8 +10,9 @@ use re_log_types::{
     example_components::{MyColor, MyIndex, MyPoint},
     EntityPath, StoreId, TimeInt, TimePoint, Timeline,
 };
-use re_types::ComponentBatch as _;
-use re_types_core::{archetypes::Clear, components::ClearIsRecursive, AsComponents as _};
+use re_types_core::{
+    archetypes::Clear, components::ClearIsRecursive, AsComponents as _, ComponentBatch as _,
+};
 
 // ---
 

crates/store/re_protos/proto/rerun/v1alpha1/remote_store.proto

@@ -57,9 +57,9 @@ service StorageNodeService {
 
   // Registration APIs
 
-  // TODO(zehiko) support registering more than one recording at a time
-  rpc RegisterRecording(RegisterRecordingRequest) returns (RegisterRecordingResponse) {}
+  rpc RegisterRecordings(RegisterRecordingsRequest) returns (RegisterRecordingsResponse) {}
 
+  // TODO(cmc): This needs to be batched. Everything needs to be batched, always.
   rpc UnregisterRecording(UnregisterRecordingRequest) returns (UnregisterRecordingResponse) {}
 
   rpc UnregisterAllRecordings(UnregisterAllRecordingsRequest) returns (UnregisterAllRecordingsResponse) {}
@@ -425,21 +425,29 @@ message GetRecordingSchemaResponse {
 
 // ---------------- RegisterRecording ------------------
 
-message RegisterRecordingRequest {
+message RegisterRecordingsRequest {
   // to which catalog entry do we want to register the recording
   CatalogEntry entry = 1;
+
+  repeated RegisterRecordingDescription recordings = 2;
+}
+
+message RegisterRecordingDescription {
   // human readable description of the recording
-  string description = 2;
+  string description = 1;
+
   // recording storage url (e.g. s3://bucket/file or file:///path/to/file)
-  string storage_url = 3;
+  string storage_url = 2;
+
   // type of recording
-  RecordingType typ = 4;
+  RecordingType typ = 3;
+
   // (optional) any additional metadata that should be associated with the recording
   // You can associate any arbtrirary number of columns with a specific recording
-  DataframePart metadata = 5;
+  DataframePart metadata = 4;
 }
 
-message RegisterRecordingResponse {
+message RegisterRecordingsResponse {
   DataframePart data = 1;
 }
 

crates/store/re_protos/src/v1alpha1/rerun.remote_store.v1alpha1.rs

@@ -824,47 +824,62 @@ impl ::prost::Name for GetRecordingSchemaResponse {
     }
 }
 #[derive(Clone, PartialEq, ::prost::Message)]
-pub struct RegisterRecordingRequest {
+pub struct RegisterRecordingsRequest {
     /// to which catalog entry do we want to register the recording
     #[prost(message, optional, tag = "1")]
     pub entry: ::core::option::Option<CatalogEntry>,
+    #[prost(message, repeated, tag = "2")]
+    pub recordings: ::prost::alloc::vec::Vec<RegisterRecordingDescription>,
+}
+impl ::prost::Name for RegisterRecordingsRequest {
+    const NAME: &'static str = "RegisterRecordingsRequest";
+    const PACKAGE: &'static str = "rerun.remote_store.v1alpha1";
+    fn full_name() -> ::prost::alloc::string::String {
+        "rerun.remote_store.v1alpha1.RegisterRecordingsRequest".into()
+    }
+    fn type_url() -> ::prost::alloc::string::String {
+        "/rerun.remote_store.v1alpha1.RegisterRecordingsRequest".into()
+    }
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct RegisterRecordingDescription {
     /// human readable description of the recording
-    #[prost(string, tag = "2")]
+    #[prost(string, tag = "1")]
     pub description: ::prost::alloc::string::String,
     /// recording storage url (e.g. s3://bucket/file or file:///path/to/file)
-    #[prost(string, tag = "3")]
+    #[prost(string, tag = "2")]
     pub storage_url: ::prost::alloc::string::String,
     /// type of recording
-    #[prost(enumeration = "RecordingType", tag = "4")]
+    #[prost(enumeration = "RecordingType", tag = "3")]
     pub typ: i32,
     /// (optional) any additional metadata that should be associated with the recording
     /// You can associate any arbtrirary number of columns with a specific recording
-    #[prost(message, optional, tag = "5")]
+    #[prost(message, optional, tag = "4")]
     pub metadata: ::core::option::Option<DataframePart>,
 }
-impl ::prost::Name for RegisterRecordingRequest {
-    const NAME: &'static str = "RegisterRecordingRequest";
+impl ::prost::Name for RegisterRecordingDescription {
+    const NAME: &'static str = "RegisterRecordingDescription";
     const PACKAGE: &'static str = "rerun.remote_store.v1alpha1";
     fn full_name() -> ::prost::alloc::string::String {
-        "rerun.remote_store.v1alpha1.RegisterRecordingRequest".into()
+        "rerun.remote_store.v1alpha1.RegisterRecordingDescription".into()
     }
     fn type_url() -> ::prost::alloc::string::String {
-        "/rerun.remote_store.v1alpha1.RegisterRecordingRequest".into()
+        "/rerun.remote_store.v1alpha1.RegisterRecordingDescription".into()
     }
 }
 #[derive(Clone, PartialEq, ::prost::Message)]
-pub struct RegisterRecordingResponse {
+pub struct RegisterRecordingsResponse {
     #[prost(message, optional, tag = "1")]
     pub data: ::core::option::Option<DataframePart>,
 }
-impl ::prost::Name for RegisterRecordingResponse {
-    const NAME: &'static str = "RegisterRecordingResponse";
+impl ::prost::Name for RegisterRecordingsResponse {
+    const NAME: &'static str = "RegisterRecordingsResponse";
     const PACKAGE: &'static str = "rerun.remote_store.v1alpha1";
     fn full_name() -> ::prost::alloc::string::String {
-        "rerun.remote_store.v1alpha1.RegisterRecordingResponse".into()
+        "rerun.remote_store.v1alpha1.RegisterRecordingsResponse".into()
     }
     fn type_url() -> ::prost::alloc::string::String {
-        "/rerun.remote_store.v1alpha1.RegisterRecordingResponse".into()
+        "/rerun.remote_store.v1alpha1.RegisterRecordingsResponse".into()
     }
 }
 #[derive(Clone, PartialEq, ::prost::Message)]
@@ -1608,26 +1623,26 @@ pub mod storage_node_service_client {
             ));
             self.inner.unary(req, path, codec).await
         }
-        /// TODO(zehiko) support registering more than one recording at a time
-        pub async fn register_recording(
+        pub async fn register_recordings(
             &mut self,
-            request: impl tonic::IntoRequest<super::RegisterRecordingRequest>,
-        ) -> std::result::Result<tonic::Response<super::RegisterRecordingResponse>, tonic::Status>
+            request: impl tonic::IntoRequest<super::RegisterRecordingsRequest>,
+        ) -> std::result::Result<tonic::Response<super::RegisterRecordingsResponse>, tonic::Status>
         {
             self.inner.ready().await.map_err(|e| {
                 tonic::Status::unknown(format!("Service was not ready: {}", e.into()))
             })?;
             let codec = tonic::codec::ProstCodec::default();
             let path = http::uri::PathAndQuery::from_static(
-                "/rerun.remote_store.v1alpha1.StorageNodeService/RegisterRecording",
+                "/rerun.remote_store.v1alpha1.StorageNodeService/RegisterRecordings",
             );
             let mut req = request.into_request();
             req.extensions_mut().insert(GrpcMethod::new(
                 "rerun.remote_store.v1alpha1.StorageNodeService",
-                "RegisterRecording",
+                "RegisterRecordings",
             ));
             self.inner.unary(req, path, codec).await
         }
+        /// TODO(cmc): This needs to be batched. Everything needs to be batched, always.
         pub async fn unregister_recording(
             &mut self,
             request: impl tonic::IntoRequest<super::UnregisterRecordingRequest>,
@@ -1791,11 +1806,11 @@ pub mod storage_node_service_server {
             &self,
             request: tonic::Request<super::GetRecordingSchemaRequest>,
         ) -> std::result::Result<tonic::Response<super::GetRecordingSchemaResponse>, tonic::Status>;
-        /// TODO(zehiko) support registering more than one recording at a time
-        async fn register_recording(
+        async fn register_recordings(
             &self,
-            request: tonic::Request<super::RegisterRecordingRequest>,
-        ) -> std::result::Result<tonic::Response<super::RegisterRecordingResponse>, tonic::Status>;
+            request: tonic::Request<super::RegisterRecordingsRequest>,
+        ) -> std::result::Result<tonic::Response<super::RegisterRecordingsResponse>, tonic::Status>;
+        /// TODO(cmc): This needs to be batched. Everything needs to be batched, always.
         async fn unregister_recording(
             &self,
             request: tonic::Request<super::UnregisterRecordingRequest>,
@@ -2524,22 +2539,23 @@ pub mod storage_node_service_server {
                     };
                     Box::pin(fut)
                 }
-                "/rerun.remote_store.v1alpha1.StorageNodeService/RegisterRecording" => {
+                "/rerun.remote_store.v1alpha1.StorageNodeService/RegisterRecordings" => {
                     #[allow(non_camel_case_types)]
-                    struct RegisterRecordingSvc<T: StorageNodeService>(pub Arc<T>);
+                    struct RegisterRecordingsSvc<T: StorageNodeService>(pub Arc<T>);
                     impl<T: StorageNodeService>
-                        tonic::server::UnaryService<super::RegisterRecordingRequest>
-                        for RegisterRecordingSvc<T>
+                        tonic::server::UnaryService<super::RegisterRecordingsRequest>
+                        for RegisterRecordingsSvc<T>
                     {
-                        type Response = super::RegisterRecordingResponse;
+                        type Response = super::RegisterRecordingsResponse;
                         type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
                         fn call(
                             &mut self,
-                            request: tonic::Request<super::RegisterRecordingRequest>,
+                            request: tonic::Request<super::RegisterRecordingsRequest>,
                         ) -> Self::Future {
                             let inner = Arc::clone(&self.0);
                             let fut = async move {
-                                <T as StorageNodeService>::register_recording(&inner, request).await
+                                <T as StorageNodeService>::register_recordings(&inner, request)
+                                    .await
                             };
                             Box::pin(fut)
                         }
@@ -2550,7 +2566,7 @@ pub mod storage_node_service_server {
                     let max_encoding_message_size = self.max_encoding_message_size;
                     let inner = self.inner.clone();
                     let fut = async move {
-                        let method = RegisterRecordingSvc(inner);
+                        let method = RegisterRecordingsSvc(inner);
                         let codec = tonic::codec::ProstCodec::default();
                         let mut grpc = tonic::server::Grpc::new(codec)
                             .apply_compression_config(

crates/top/rerun/src/commands/entrypoint.rs

@@ -809,14 +809,12 @@ fn run_impl(
             anyhow::bail!("`--serve` does not support catalogs");
         }
 
-        #[cfg(not(feature = "server"))]
-        {
+        if !cfg!(feature = "server") {
             _ = (call_source, rxs);
             anyhow::bail!("Can't host server - rerun was not compiled with the 'server' feature");
         }
 
-        #[cfg(not(feature = "web_viewer"))]
-        if args.web_viewer {
+        if !cfg!(feature = "web_viewer") {
             anyhow::bail!(
                 "Can't host web-viewer - rerun was not compiled with the 'web_viewer' feature"
             );

crates/utils/re_arrow_util/Cargo.toml

@@ -28,3 +28,6 @@ re_tracing.workspace = true
 
 arrow.workspace = true
 itertools.workspace = true
+
+[dev-dependencies]
+insta.workspace = true