SDK DataLoaders 8: customizable (external) loaders for C++ (#5361)

Introduces the new `DataLoaderSettings` business to C++ and update
examples accordingly (`external_data_loader` & `log_file`).

```bash
./build/debug/examples/cpp/log_file/example_log_file --recording-id this-one --entity-path-prefix a/b/c  --time sim_time=1000 --time wall_time=1709204046 --sequence sim_frame=42 rerun_cpp/tests/main.cpp | rerun -
```

![image](https://github.com/rerun-io/rerun/assets/2910679/b979a24c-29b6-473b-91b1-de3832bea436)


Checks:
- [x] external loader ran manually (`loader.exe | rerun`)
- [x] external loader via rerun (`rerun xxx.cpp`)
- [x] log_file with external loader (`log_file xxx.cpp`)

---

Part of series of PR to expose configurable `DataLoader`s to our SDKs:
- #5327 
- #5328 
- #5330
- #5337
- #5351
- #5355
- #5379
- #5361
- #5388

Author: teh-cmc

crates/rerun_c/src/lib.rs

@@ -731,13 +731,16 @@ pub unsafe extern "C" fn rr_recording_stream_log(
 fn rr_log_file_from_path_impl(
     stream: CRecordingStream,
     filepath: CStringView,
+    entity_path_prefix: CStringView,
+    timeless: bool,
 ) -> Result<(), CError> {
     let stream = recording_stream(stream)?;
 
     let filepath = filepath.as_str("filepath")?;
+    let entity_path_prefix = entity_path_prefix.as_str("entity_path_prefix").ok();
+
     stream
-        // TODO(cmc): expose settings
-        .log_file_from_path(filepath, None, true)
+        .log_file_from_path(filepath, entity_path_prefix.map(Into::into), timeless)
         .map_err(|err| {
             CError::new(
                 CErrorCode::RecordingStreamRuntimeFailure,
@@ -753,9 +756,11 @@ fn rr_log_file_from_path_impl(
 pub unsafe extern "C" fn rr_recording_stream_log_file_from_path(
     stream: CRecordingStream,
     filepath: CStringView,
+    entity_path_prefix: CStringView,
+    timeless: bool,
     error: *mut CError,
 ) {
-    if let Err(err) = rr_log_file_from_path_impl(stream, filepath) {
+    if let Err(err) = rr_log_file_from_path_impl(stream, filepath, entity_path_prefix, timeless) {
         err.write_error(error);
     }
 }
@@ -766,15 +771,22 @@ fn rr_log_file_from_contents_impl(
     stream: CRecordingStream,
     filepath: CStringView,
     contents: CBytesView,
+    entity_path_prefix: CStringView,
+    timeless: bool,
 ) -> Result<(), CError> {
     let stream = recording_stream(stream)?;
 
     let filepath = filepath.as_str("filepath")?;
     let contents = contents.as_bytes("contents")?;
+    let entity_path_prefix = entity_path_prefix.as_str("entity_path_prefix").ok();
 
     stream
-        // TODO(cmc): expose settings
-        .log_file_from_contents(filepath, std::borrow::Cow::Borrowed(contents), None, true)
+        .log_file_from_contents(
+            filepath,
+            std::borrow::Cow::Borrowed(contents),
+            entity_path_prefix.map(Into::into),
+            timeless,
+        )
         .map_err(|err| {
             CError::new(
                 CErrorCode::RecordingStreamRuntimeFailure,
@@ -791,9 +803,13 @@ pub unsafe extern "C" fn rr_recording_stream_log_file_from_contents(
     stream: CRecordingStream,
     filepath: CStringView,
     contents: CBytesView,
+    entity_path_prefix: CStringView,
+    timeless: bool,
     error: *mut CError,
 ) {
-    if let Err(err) = rr_log_file_from_contents_impl(stream, filepath, contents) {
+    if let Err(err) =
+        rr_log_file_from_contents_impl(stream, filepath, contents, entity_path_prefix, timeless)
+    {
         err.write_error(error);
     }
 }

crates/rerun_c/src/rerun.h

@@ -141,6 +141,30 @@ typedef struct rr_spawn_options {
     rr_string executable_path;
 } rr_spawn_options;
 
+/// Recommended settings for the [`DataLoader`].
+///
+/// The loader is free to ignore some or all of these.
+///
+/// Refer to the field-level documentation for more information about each individual options.
+//
+// TODO(#3841): expose timepoint settings once we implement stateless APIs
+typedef struct rr_data_loader_settings {
+    /// The recommended `RecordingId` to log the data to.
+    ///
+    /// Unspecified by default.
+    rr_string recording_id;
+
+    /// What should the logged entity paths be prefixed with?
+    ///
+    /// Unspecified by default.
+    rr_string entity_path_prefix;
+
+    /// Should the logged data be timeless?
+    ///
+    /// Defaults to `false` if not set.
+    bool timeless;
+} rr_data_loader_settings;
+
 typedef struct rr_store_info {
     /// The user-chosen name of the application doing the logging.
     rr_string application_id;
@@ -432,7 +456,8 @@ extern void rr_recording_stream_log(
 ///
 /// See <https://www.rerun.io/docs/howto/open-any-file> for more information.
 extern void rr_recording_stream_log_file_from_path(
-    rr_recording_stream stream, rr_string path, rr_error* error
+    rr_recording_stream stream, rr_string path, rr_string entity_path_prefix, bool timeless,
+    rr_error* error
 );
 
 /// Logs the given `contents` using all `DataLoader`s available.
@@ -444,7 +469,8 @@ extern void rr_recording_stream_log_file_from_path(
 ///
 /// See <https://www.rerun.io/docs/howto/open-any-file> for more information.
 extern void rr_recording_stream_log_file_from_contents(
-    rr_recording_stream stream, rr_string path, rr_bytes contents, rr_error* error
+    rr_recording_stream stream, rr_string path, rr_bytes contents, rr_string entity_path_prefix,
+    bool timeless, rr_error* error
 );
 
 // ----------------------------------------------------------------------------

examples/cpp/external_data_loader/main.cpp

@@ -1,11 +1,55 @@
+#include <cstdint>
 #include <fstream>
 #include <iostream>
+#include <iterator>
 #include <sstream>
 #include <string>
 
 #include <rerun.hpp>
+#include <rerun/third_party/cxxopts.hpp>
+#include <string_view>
+
+void set_time_from_args(const rerun::RecordingStream& rec, cxxopts::ParseResult& args) {
+    if (args.count("time")) {
+        const auto times = args["time"].as<std::vector<std::string>>();
+        for (const auto& time_str : times) {
+            auto pos = time_str.find('=');
+            if (pos != std::string::npos) {
+                auto timeline_name = time_str.substr(0, pos);
+                int64_t time = std::stol(time_str.substr(pos + 1));
+                rec.set_time_seconds(timeline_name, static_cast<double>(time));
+            }
+        }
+    }
+
+    if (args.count("sequence")) {
+        const auto sequences = args["sequence"].as<std::vector<std::string>>();
+        for (const auto& sequence_str : sequences) {
+            auto pos = sequence_str.find('=');
+            if (pos != std::string::npos) {
+                auto timeline_name = sequence_str.substr(0, pos);
+                int64_t sequence = std::stol(sequence_str.substr(pos + 1));
+                rec.set_time_sequence(timeline_name, sequence);
+            }
+        }
+    }
+}
 
-static const char* USAGE = R"(
+int main(int argc, char* argv[]) {
+    // The Rerun Viewer will always pass these two pieces of information:
+    // 1. The path to be loaded, as a positional arg.
+    // 2. A shared recording ID, via the `--recording-id` flag.
+    //
+    // It is up to you whether you make use of that shared recording ID or not.
+    // If you use it, the data will end up in the same recording as all other plugins interested in
+    // that file, otherwise you can just create a dedicated recording for it. Or both.
+    //
+    // Check out `re_data_source::DataLoaderSettings` documentation for an exhaustive listing of
+    // the available CLI parameters.
+
+    cxxopts::Options options(
+        "rerun-loader-cpp-file",
+        R"(
 This is an example executable data-loader plugin for the Rerun Viewer.
 Any executable on your `$PATH` with a name that starts with `rerun-loader-` will be treated as an
 external data-loader.
@@ -15,57 +59,38 @@ special exit code to indicate that it doesn't support anything else.
 
 To try it out, compile it and place it in your $PATH as `rerun-loader-cpp-file`, then open a C++ source
 file with Rerun (`rerun file.cpp`).
+)"
+    );
 
-USAGE:
-  rerun-loader-cpp-file [OPTIONS] FILEPATH
-
-FLAGS:
-  -h, --help                    Prints help information
-
-OPTIONS:
-  --recording-id RECORDING_ID   ID of the shared recording
-
-ARGS:
-  <FILEPATH>
-)";
-
-int main(int argc, char* argv[]) {
-    // The Rerun Viewer will always pass these two pieces of information:
-    // 1. The path to be loaded, as a positional arg.
-    // 2. A shared recording ID, via the `--recording-id` flag.
-    //
-    // It is up to you whether you make use of that shared recording ID or not.
-    // If you use it, the data will end up in the same recording as all other plugins interested in
-    // that file, otherwise you can just create a dedicated recording for it. Or both.
-    std::string filepath;
-    std::string recording_id;
-
-    for (int i = 1; i < argc; ++i) {
-        std::string arg(argv[i]);
-
-        if (arg == "--recording-id") {
-            if (i + 1 < argc) {
-                recording_id = argv[i + 1];
-                ++i;
-            } else {
-                std::cerr << USAGE << std::endl;
-                return 1;
-            }
-        } else {
-            filepath = arg;
-        }
+    // clang-format off
+    options.add_options()
+      ("h,help", "Print usage")
+      ("filepath", "The filepath to be loaded and logged", cxxopts::value<std::string>())
+      ("application-id", "Optional recommended ID for the application", cxxopts::value<std::string>())
+      ("recording-id", "Optional recommended ID for the recording", cxxopts::value<std::string>())
+      ("entity-path-prefix", "Optional prefix for all entity paths", cxxopts::value<std::string>())
+      ("timeless", "Optionally mark data to be logged as timeless", cxxopts::value<bool>()->default_value("false"))
+      ("time", "Optional timestamps to log at (e.g. `--time sim_time=1709203426`) (repeatable)", cxxopts::value<std::vector<std::string>>())
+      ("sequence", "Optional sequences to log at (e.g. `--sequence sim_frame=42`) (repeatable)", cxxopts::value<std::vector<std::string>>())
+    ;
+    // clang-format on
+
+    options.parse_positional({"filepath"});
+
+    auto args = options.parse(argc, argv);
+
+    if (args.count("help")) {
+        std::cout << options.help() << std::endl;
+        exit(0);
     }
 
-    if (filepath.empty()) {
-        std::cerr << USAGE << std::endl;
-        return 1;
-    }
+    const auto filepath = args["filepath"].as<std::string>();
 
     bool is_file = std::filesystem::is_regular_file(filepath);
     bool is_cpp_file = std::filesystem::path(filepath).extension().string() == ".cpp";
 
     // Inform the Rerun Viewer that we do not support that kind of file.
-    if (!is_file || is_cpp_file) {
+    if (!is_file || !is_cpp_file) {
         return rerun::EXTERNAL_DATA_LOADER_INCOMPATIBLE_EXIT_CODE;
     }
 
@@ -75,12 +100,27 @@ int main(int argc, char* argv[]) {
 
     std::string text = "## Some C++ code\n```cpp\n" + body.str() + "\n```\n";
 
-    const auto rec = rerun::RecordingStream("rerun_example_external_data_loader", recording_id);
+    auto application_id = std::string_view("rerun_example_external_data_loader");
+    if (args.count("application-id")) {
+        application_id = args["application-id"].as<std::string>();
+    }
+    auto recording_id = std::string_view();
+    if (args.count("recording-id")) {
+        recording_id = args["recording-id"].as<std::string>();
+    }
+    const auto rec = rerun::RecordingStream(application_id, recording_id);
     // The most important part of this: log to standard output so the Rerun Viewer can ingest it!
     rec.to_stdout().exit_on_failure();
 
-    rec.log_timeless(
-        filepath,
+    set_time_from_args(rec, args);
+
+    auto entity_path = std::string(filepath);
+    if (args.count("entity-path-prefix")) {
+        entity_path = args["entity-path-prefix"].as<std::string>() + "/" + filepath;
+    }
+    rec.log_with_timeless(
+        entity_path,
+        args["timeless"].as<bool>(),
         rerun::TextDocument(text).with_media_type(rerun::MediaType::markdown())
     );
 }

examples/cpp/log_file/main.cpp

@@ -58,7 +58,7 @@ int main(int argc, char** argv) {
         for (const auto& filepath : filepaths) {
             if (!from_contents) {
                 // Either log the file using its path…
-                rec.log_file_from_path(filepath);
+                rec.log_file_from_path(filepath, "log_file_example");
             } else {
                 // …or using its contents if you already have them loaded for some reason.
                 if (std::filesystem::is_regular_file(filepath)) {
@@ -70,7 +70,8 @@ int main(int argc, char** argv) {
                     rec.log_file_from_contents(
                         filepath,
                         reinterpret_cast<const std::byte*>(data.c_str()),
-                        data.size()
+                        data.size(),
+                        "log_file_example"
                     );
                 }
             }