Merge branch 'main' into antoine/filt9-slash-in-query

Author: abey79

.github/workflows/contrib_checks.yml

@@ -111,7 +111,7 @@ jobs:
         with:
           vulkan_version: ${{ env.VULKAN_SDK_VERSION }}
           install_runtime: true
-          cache: false
+          cache: true
           stripdown: true
 
       - name: Setup software rasterizer

.github/workflows/reusable_checks_rust.yml

@@ -79,7 +79,7 @@ jobs:
         with:
           vulkan_version: ${{ env.VULKAN_SDK_VERSION }}
           install_runtime: true
-          cache: false
+          cache: true
           stripdown: true
 
       - name: Setup software rasterizer

.gitignore

@@ -34,6 +34,7 @@ __pycache__
 *.pyd
 *.so
 **/.pytest_cache
+**/.ipynb_checkpoints/
 
 # Pixi environment
 .pixi

.vscode/settings.json

@@ -8,6 +8,11 @@
     "files.autoGuessEncoding": true,
     "files.insertFinalNewline": true,
     "files.trimTrailingWhitespace": true,
+    "protoc": {
+        "options": [
+            "--proto_path=crates/store/re_protos/proto",
+        ]
+    },
     // Whether `--workspace` should be passed to `cargo check`. If false, `-p <package>` will be passed instead.
     "rust-analyzer.check.workspace": false,
     "rust-analyzer.cargo.allTargets": true,
@@ -42,12 +47,10 @@
     },
     "rust-analyzer.diagnostics.enable": false, // False positives
     "rust-analyzer.showUnlinkedFileNotification": false,
-
     // Uncomment the following options and restart rust-analyzer to get it to check code behind `cfg(target_arch=wasm32)`.
     // Don't forget to put it in a comment again before committing.
     // "rust-analyzer.cargo.target": "wasm32-unknown-unknown",
     // "rust-analyzer.cargo.cfgs": ["web","webgl","webgpu"],
-
     "C_Cpp.default.configurationProvider": "ms-vscode.cmake-tools", // Use cmake-tools to grab configs.
     "C_Cpp.autoAddFileAssociations": false,
     "cmake.buildDirectory": "${workspaceRoot}/build/debug",
@@ -63,6 +66,9 @@
     ],
     "prettier.requireConfig": true,
     "prettier.configPath": ".prettierrc.toml",
+    "[proto3]": {
+        "editor.defaultFormatter": "zxh404.vscode-proto3"
+    },
     "[python]": {
         "editor.defaultFormatter": "charliermarsh.ruff"
     },

CHANGELOG.md

@@ -1,6 +1,139 @@
 # Rerun changelog
 
-## [Unreleased](https://github.com/rerun-io/rerun/compare/latest...HEAD)
+## [Unreleased](https://github.com/rerun-io/rerun/compare/0.21.0...HEAD) - Entity filter & improved partial update API
+
+TODO(andreas): add link to release video
+
+📖 Release blogpost: TODO(andreas): add link
+
+🧳 Migration guide: https://rerun.io/docs/reference/migration/migration-0-22?speculative-link
+
+### ✨ Overview & highlights
+
+#### Viewer
+* 🔎 Added entity filtering/searching
+* 🔔 Recent notifications show now in a dedicated panel
+* 🖱️ Entity ranges can now be selected with shift + click
+
+#### APIs
+* 🔄 [Much easier partial updates of archetypes](https://rerun.io/docs/howto/logging/send-partial-updates?speculative-link)
+* 📊 [Greatly improved ease of use of `send_columns`](https://rerun.io/docs/howto/logging/send-columns?speculative-link)
+* ⏱️ Python notebooks & JS can now control the timeline with `set_time_ctrl` TODO(#8894): link to the api/an example
+* 📝 Lots of [new snippets](https://github.com/rerun-io/rerun/blob/0.22.0/docs/snippets/INDEX.md?speculative-link) for demonstrating partial updates & custom data logging in Python/C++/Rust
+
+The API & related under-the-hood changes pave the way for better support for multiple archetypes on the same entity and components with generic types in future releases.
+Stay tuned!
+
+### ⚠️ Breaking changes
+TODO(andreas): fill in
+🧳 Migration guide: https://rerun.io/docs/reference/migration/migration-0-22?speculative-link
+
+### 🔎 Details
+
+#### 🪵 Log API
+- Tensor shape and dimension names are now separate arrow fields [#8376](https://github.com/rerun-io/rerun/pull/8376)
+- Remove deprecated `DisconnectedSpace` archetype & component [#8545](https://github.com/rerun-io/rerun/pull/8545)
+- Add `any_values` and `extra_values` snippets for rust [#8718](https://github.com/rerun-io/rerun/pull/8718)
+- Implement gRPC log sink [#8709](https://github.com/rerun-io/rerun/pull/8709)
+- Implement gRPC log stream [#8730](https://github.com/rerun-io/rerun/pull/8730)
+
+#### 🌊 C++ API
+- Fix compilation for GCC 13.3 (missing `cstdint` include) [#8609](https://github.com/rerun-io/rerun/pull/8609) (thanks [@plumonito](https://github.com/plumonito)!)
+- Introduce eager serialization & update/clear APIs in the C++ SDK [#8727](https://github.com/rerun-io/rerun/pull/8727)
+- Make all C++ archetypes eager serialized & provide generated update/clear APIs [#8779](https://github.com/rerun-io/rerun/pull/8779)
+- C++ `columns` method for convenient `send_columns` call through archetypes [#8828](https://github.com/rerun-io/rerun/pull/8828)
+- Add `with_many_` variants for C++ archetype mono fields & port remaining snippets [#8836](https://github.com/rerun-io/rerun/pull/8836)
+- Require descriptors to be provided on all log calls in C++ (either explicitly or implicitly via archetype) [#8853](https://github.com/rerun-io/rerun/pull/8853)
+- Deprecate C++ `TimeColumn::from_sequence_points` in favor of `TimeColumn::from_sequence` [#8882](https://github.com/rerun-io/rerun/pull/8882)
+- `AsComponents::serialize` is now `AsComponents::as_batches` and returns `Collection<ComponentBatch>` [#8884](https://github.com/rerun-io/rerun/pull/8884)
+- Make it easy to log custom arrow data in C++ [#8880](https://github.com/rerun-io/rerun/pull/8880)
+- Rerun CMake dependency now automatically ensures C++17 or newer [#8898](https://github.com/rerun-io/rerun/pull/8898)
+
+#### 🐍 Python API
+- Autogenerated partial updates APIs for Python [#8671](https://github.com/rerun-io/rerun/pull/8671)
+- Remove unused `num_instances()` method [#8702](https://github.com/rerun-io/rerun/pull/8702)
+- Tagged columnar updates: Python [#8792](https://github.com/rerun-io/rerun/pull/8792)
+- Include a python API for routing time control commands to the notebook instance [#8809](https://github.com/rerun-io/rerun/pull/8809)
+- Python: remove legacy `send_columns` and update everything left [#8799](https://github.com/rerun-io/rerun/pull/8799)
+- Deprecate Python's `log_components` [#8892](https://github.com/rerun-io/rerun/pull/8892)
+
+#### 🦀 Rust API
+- Update MSRV to 1.81 [#8529](https://github.com/rerun-io/rerun/pull/8529)
+- Fix `RecordingStream::log` implicitly requiring `Sized` [#8587](https://github.com/rerun-io/rerun/pull/8587)
+- Add example for extending the viewer with custom callbacks [#8284](https://github.com/rerun-io/rerun/pull/8284)
+- `EntityPathFilter` variable substitutions are now delegated to (new) `ResolvedEntityPathFilter` [#8543](https://github.com/rerun-io/rerun/pull/8543)
+- Specify, test, and fix all broken `AsComponents`<>`ComponentBatch` interactions from blanket impls [#8591](https://github.com/rerun-io/rerun/pull/8591)
+- New types and traits for (co-existing!) eager serialization [#8642](https://github.com/rerun-io/rerun/pull/8642)
+- Autogenerate tagging-compliant descriptor methods for all archetypes [#8643](https://github.com/rerun-io/rerun/pull/8643)
+- Automatically generate partial update APIs for eager archetypes [#8647](https://github.com/rerun-io/rerun/pull/8647)
+- Tagged columnar updates: Rust [#8764](https://github.com/rerun-io/rerun/pull/8764)
+- Make  `Box`/`AssetVideo`/`ViewCoordinates`/`Asset3D` eager serialized in Rust [#8785](https://github.com/rerun-io/rerun/pull/8785)
+- Make `Pinhole` archetype in Rust eager serialized [#8789](https://github.com/rerun-io/rerun/pull/8789)
+- Make `Image` & `Mesh3D` archetypes in Rust eager serialized [#8793](https://github.com/rerun-io/rerun/pull/8793)
+- Make rust `Tensor` archetype eager serialized [#8801](https://github.com/rerun-io/rerun/pull/8801)
+- Rust: remove legacy `send_columns` and update everything left [#8804](https://github.com/rerun-io/rerun/pull/8804)
+- `ComponentBatch` doesn't implement `AsComponents` anymore [#8820](https://github.com/rerun-io/rerun/pull/8820)
+
+#### 🪳 Bug fixes
+- Fix WSL support, update troubleshooting guide [#8610](https://github.com/rerun-io/rerun/pull/8610)
+- Handle empty line strips in the viewer [#8653](https://github.com/rerun-io/rerun/pull/8653)
+- Fix clicking of links in markdown [#8794](https://github.com/rerun-io/rerun/pull/8794)
+- Fix CPU spike caused by hanging connection after socket closure (#8806) [#8810](https://github.com/rerun-io/rerun/pull/8810) (thanks [@goktug97](https://github.com/goktug97)!)
+- Make it possible to change the contents of a view multiple times per frame [#8854](https://github.com/rerun-io/rerun/pull/8854)
+- Fix playback issues with some h264 videos on native & Safari [#8850](https://github.com/rerun-io/rerun/pull/8850)
+- Fix handling null timestamps in the dataframe [#8897](https://github.com/rerun-io/rerun/pull/8897)
+
+#### 🌁 Viewer improvements
+- Remove all legacy Chunk iteration APIs [#8556](https://github.com/rerun-io/rerun/pull/8556)
+- Implement copy-screenshot-to-clipboard on Web [#8607](https://github.com/rerun-io/rerun/pull/8607)
+- Improve transform performance (by caching affine transforms resulting from transform components) [#8691](https://github.com/rerun-io/rerun/pull/8691)
+
+#### 🧑‍🏫 Examples
+- add prompt depth anything example [#8888](https://github.com/rerun-io/rerun/pull/8888) (thanks [@pablovela5620](https://github.com/pablovela5620)!)
+
+#### 📚 Docs
+- Add new `Transform3D` partial updates snippet for all languages [#8690](https://github.com/rerun-io/rerun/pull/8690)
+- doc: Update `annotation-context.rs` to use correct API [#8708](https://github.com/rerun-io/rerun/pull/8708) (thanks [@OlivierLDff](https://github.com/OlivierLDff)!)
+
+#### 🖼 UI improvements
+- Show the `GraphNode` as a label by default [#8542](https://github.com/rerun-io/rerun/pull/8542)
+- Short circuit graph simulation if all nodes are fixed [#8549](https://github.com/rerun-io/rerun/pull/8549)
+- Panel with recent notifications [#8465](https://github.com/rerun-io/rerun/pull/8465)
+- Fix tooltips being dragged along in graph view [#8573](https://github.com/rerun-io/rerun/pull/8573)
+- Restore the time panel help button [#8599](https://github.com/rerun-io/rerun/pull/8599)
+- Filter entities in the UI (part 0): Make `CustomContent` more useful [#8645](https://github.com/rerun-io/rerun/pull/8645)
+- Filter entities in the UI (part 1): Introduce a filter widget [#8652](https://github.com/rerun-io/rerun/pull/8652)
+- Filter entities in the UI (part 2): Introduce entity filtering in the time panel [#8654](https://github.com/rerun-io/rerun/pull/8654)
+- Filter entities in the UI (part 3): Move action to a menu in the blueprint panel and keep default blueprint when using heuristics [#8672](https://github.com/rerun-io/rerun/pull/8672)
+- Filter entities in the UI (part 4): Add entity filtering in the blueprint tree [#8706](https://github.com/rerun-io/rerun/pull/8706)
+- Draw nodes above edges in graph view [#8738](https://github.com/rerun-io/rerun/pull/8738)
+- Filter entities in the UI (part 5): Add snapshot tests for the blueprint tree [#8728](https://github.com/rerun-io/rerun/pull/8728)
+- Filter entities in the UI (part 6): Refactor `re_blueprint_tree` and add more tests [#8795](https://github.com/rerun-io/rerun/pull/8795)
+- Improve performance for Blueprint & Streams Panel for many entities [#8808](https://github.com/rerun-io/rerun/pull/8808)
+- The empty/full entity icon now reflects presence of component on the current timeline [#8839](https://github.com/rerun-io/rerun/pull/8839)
+- Show start of large arrow values instead of just their size [#8861](https://github.com/rerun-io/rerun/pull/8861)
+- Implement range selection with shift-click in the blueprint tree [#8852](https://github.com/rerun-io/rerun/pull/8852)
+- Filter entities in the UI (part 7): Refactor and optimize `re_time_panel` and add more tests [#8863](https://github.com/rerun-io/rerun/pull/8863)
+- Clickable URLs in named components / `AnyValues` [#8864](https://github.com/rerun-io/rerun/pull/8864)
+- Implement range selection with shift-click in the time panel [#8870](https://github.com/rerun-io/rerun/pull/8870)
+- Improve look of bar charts slightly [#8875](https://github.com/rerun-io/rerun/pull/8875)
+
+#### 🕸️ Web
+- Add JS timeline control and callback APIs [#8673](https://github.com/rerun-io/rerun/pull/8673)
+
+#### ✨ Other enhancement
+- Remove `Chunk::iter_component_arrays` [#8548](https://github.com/rerun-io/rerun/pull/8548)
+- Introduce new Chunk iteration APIs [#8553](https://github.com/rerun-io/rerun/pull/8553)
+
+#### 📈 Analytics
+- Add analytics for wgpu backend and whether the viewer runs in WSL [#8612](https://github.com/rerun-io/rerun/pull/8612)
+
+#### 📦 Dependencies
+- Update wgpu to 24.0.0 [#8743](https://github.com/rerun-io/rerun/pull/8743)
+
+#### 🤷‍ Other
+- Introduce snapshot unit tests for `re_component_ui` [#8546](https://github.com/rerun-io/rerun/pull/8546)
+
 
 ## [0.21.0](https://github.com/rerun-io/rerun/compare/0.20.3...0.21.0) - Graph view, 3D Grid & UI/UX improvements
 

Cargo.lock

@@ -6899,6 +6899,7 @@ dependencies = [
  "image",
  "itertools 0.13.0",
  "mimalloc",
+ "ndarray",
  "nohash-hasher",
  "once_cell",
  "ordered-float",
@@ -6918,6 +6919,7 @@ dependencies = [
  "re_video",
  "re_view",
  "re_viewer_context",
+ "re_viewport",
  "re_viewport_blueprint",
  "serde",
  "smallvec",
@@ -7102,7 +7104,6 @@ dependencies = [
  "egui_kittest",
  "egui_tiles",
  "emath",
- "env_logger",
  "glam",
  "half",
  "home",

crates/store/re_protos/proto/rerun/v0/remote_store.proto

@@ -30,7 +30,8 @@ service StorageNode {
     rpc RegisterRecording(RegisterRecordingRequest) returns (DataframePart) {}
 
     rpc UnregisterRecording(UnregisterRecordingRequest) returns (UnregisterRecordingResponse) {}
-    rpc UnregisterAllRecordings(UnregisterAllRecordingsRequest) returns (UnregisterAllRecordingsResponse) {}
+    rpc UnregisterAllRecordings(UnregisterAllRecordingsRequest)
+        returns (UnregisterAllRecordingsResponse) {}
 }
 
 // ---------------- Common response message ------------------
@@ -83,7 +84,7 @@ message IndexProperties {
 
 message InvertedIndex {
     bool store_position = 1;
-    string base_tokenizer  = 2;
+    string base_tokenizer = 2;
     // TODO(zehiko) add other properties as needed
 }
 
@@ -108,7 +109,6 @@ message CreateIndexResponse {
     uint64 indexed_rows = 1;
 }
 
-
 // ---------------- SearchIndex ------------------
 
 message SearchIndexRequest {
@@ -188,7 +188,6 @@ message UnregisterRecordingResponse {}
 message UnregisterAllRecordingsRequest {}
 message UnregisterAllRecordingsResponse {}
 
-
 // ---------------- UpdateCatalog  -----------------
 
 message UpdateCatalogRequest {

crates/top/re_sdk/src/log_sink.rs

@@ -31,6 +31,7 @@ pub trait LogSink: Send + Sync + 'static {
 
     /// Blocks until all pending data in the sink's send buffers has been fully flushed.
     ///
+    /// If applicable, this should flush all data to any underlying OS-managed file descriptors.
     /// See also [`LogSink::drop_if_disconnected`].
     fn flush_blocking(&self);
 

crates/top/re_sdk/src/recording_stream.rs

@@ -696,9 +696,9 @@ impl RecordingStreamBuilder {
 ///   thread originally sent them in, from its point of view.
 /// - There isn't any well defined global order across multiple threads.
 ///
-/// This means that e.g. flushing the pipeline ([`Self::flush_blocking`]) guarantees that all
-/// previous data sent by the calling thread has been recorded; no more, no less.
-/// (e.g. it does not mean that all file caches are flushed)
+/// This means that e.g. flushing the pipeline ([`Self::flush_blocking`]) guarantees that all previous data sent by the calling thread
+/// has been recorded and (if applicable) flushed to the underlying OS-managed file descriptor,
+/// but other threads may still have data in flight.
 ///
 /// ## Shutdown
 ///

crates/utils/re_log/src/lib.rs

@@ -95,7 +95,7 @@ pub fn default_log_filter() -> String {
         if cfg!(debug_assertions) {
             "debug".to_owned()
         } else {
-            "info".to_owned()
+            "warn".to_owned()
         }
     });
 

crates/viewer/re_blueprint_tree/src/blueprint_tree.rs

@@ -99,7 +99,7 @@ impl BlueprintTree {
                     .menu_button(&re_ui::icons::MORE, |ui| {
                         add_new_view_or_container_menu_button(ctx, viewport_blueprint, ui);
                         set_blueprint_to_default_menu_buttons(ctx, ui);
-                        set_blueprint_to_auto_menu_button(ctx, viewport_blueprint, ui);
+                        set_blueprint_to_auto_menu_button(ctx, ui);
                     }),
                 );
             });
@@ -1202,12 +1202,13 @@ fn set_blueprint_to_default_menu_buttons(ctx: &ViewerContext<'_>, ui: &mut egui:
     }
 }
 
-fn set_blueprint_to_auto_menu_button(
-    ctx: &ViewerContext<'_>,
-    viewport: &ViewportBlueprint,
-    ui: &mut egui::Ui,
-) {
-    let enabled = !viewport.auto_layout() || !viewport.auto_views();
+fn set_blueprint_to_auto_menu_button(ctx: &ViewerContext<'_>, ui: &mut egui::Ui) {
+    // Figuring out when resetting to heuristic blueprint is not changing anything is actually quite hard.
+    // There's a wide variety of things to consider that aren't easily caught:
+    // * does running view-generation/layout-generation change anything?
+    //    * these heuristics run incrementally, does rerunning them in bulk change anything?
+    // * any changes in overrides/defaults/view-property means that a reset would change something
+    let enabled = true;
 
     if ui
         .add_enabled(

crates/viewer/re_time_panel/Cargo.toml

@@ -20,7 +20,11 @@ all-features = true
 
 [features]
 default = []
-testing = ["re_viewer_context/testing", "re_viewport_blueprint/testing"]
+testing = [
+  "re_viewer_context/testing",
+  "re_viewport_blueprint/testing",
+  "smallvec/serde",
+]
 
 [dependencies]
 re_chunk_store.workspace = true

crates/viewer/re_view_spatial/Cargo.toml

@@ -64,8 +64,13 @@ web-time.workspace = true
 
 
 [dev-dependencies]
+re_viewer_context = { workspace = true, features = ["testing"] }
+re_viewport.workspace = true
+re_viewport_blueprint = { workspace = true, features = ["testing"] }
+
 criterion.workspace = true
 mimalloc.workspace = true
+ndarray.workspace = true
 
 [lib]
 bench = false