Auto create server collection

stac-cli/CHANGELOG.md

@@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Added
 
 - Geoparquet support ([#300](https://github.com/stac-utils/stac-rs/pull/300))
+- Auto-create collections when serving ([#304](https://github.com/stac-utils/stac-rs/pull/304))
 
 ## [0.2.0] - 2024-08-12
 

stac-cli/src/args.rs

@@ -151,6 +151,10 @@ pub struct ServeArgs {
     /// The pgstac connection string.
     #[arg(long)]
     pub pgstac: Option<String>,
+
+    /// Don't auto-create collections for items that are missing them.
+    #[arg(short, long)]
+    pub dont_auto_create_collections: bool,
 }
 
 /// Arguments for sorting a STAC value.

stac-cli/src/subcommand/serve.rs

@@ -12,7 +12,9 @@ impl Subcommand {
             {
                 let mut backend = stac_server::PgstacBackend::new_from_stringlike(pgstac).await?;
                 if !args.href.is_empty() {
-                    backend.add_from_hrefs(&args.href).await?;
+                    backend
+                        .add_from_hrefs(&args.href, !args.dont_auto_create_collections)
+                        .await?;
                 }
                 let api = Api::new(backend, root)?;
                 let router = stac_server::routes::from_api(api);
@@ -29,7 +31,9 @@ impl Subcommand {
         } else {
             let mut backend = MemoryBackend::new();
             if !args.href.is_empty() {
-                backend.add_from_hrefs(&args.href).await?;
+                backend
+                    .add_from_hrefs(&args.href, !args.dont_auto_create_collections)
+                    .await?;
             }
             let api = Api::new(backend, root)?;
             let router = stac_server::routes::from_api(api);

stac-server/CHANGELOG.md

@@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased]
 
+### Added
+
+- Auto-create collections on ingest ([#304](https://github.com/stac-utils/stac-rs/pull/304))
+
 ## [0.1.1] - 2024-08-12
 
 ### Added

stac-server/src/backend/mod.rs

@@ -7,8 +7,9 @@ use async_trait::async_trait;
 pub use memory::MemoryBackend;
 #[cfg(feature = "pgstac")]
 pub use pgstac::PgstacBackend;
-use stac::{Collection, Item, Value};
+use stac::{Collection, Item, Links, Value};
 use stac_api::{ItemCollection, Items, Search};
+use std::collections::{HashMap, HashSet};
 
 /// Storage backend for a STAC API.
 #[async_trait]
@@ -26,7 +27,9 @@ pub trait Backend: Clone + Sync + Send + 'static {
 
     /// Adds collections and items from hrefs.
     ///
-    /// A default implementation is provided.
+    /// A default implementation is provided. If `auto_create_collections` is
+    /// true, then, _if_ there is no collection for one or more items, a
+    /// collection will be auto-created before adding the items.
     ///
     /// # Examples
     ///
@@ -37,29 +40,70 @@ pub trait Backend: Clone + Sync + Send + 'static {
     /// backend.add_from_hrefs(&[
     ///     "tests/data/collection.json".to_string(),
     ///     "tests/data/feature.geojson".to_string(),
-    /// ]);
+    /// ], false);
     /// # })
     /// ```
-    async fn add_from_hrefs(&mut self, hrefs: &[String]) -> Result<()> {
-        let mut items = Vec::new();
+    async fn add_from_hrefs(
+        &mut self,
+        hrefs: &[String],
+        auto_create_collections: bool,
+    ) -> Result<()> {
+        let mut items: HashMap<Option<String>, Vec<Item>> = HashMap::new();
+        let mut item_collection_ids = HashSet::new();
+        let mut collection_ids = HashSet::new();
         for href in hrefs {
             let value: Value = stac_async::read(href).await?;
             match value {
-                Value::Item(item) => items.push(item),
+                Value::Item(mut item) => {
+                    item.remove_structural_links();
+                    if let Some(collection) = item.collection.as_ref() {
+                        let collection = collection.clone();
+                        let _ = item_collection_ids.insert(collection.clone());
+                        let _ = items.entry(Some(collection)).or_default().push(item);
+                    } else {
+                        let _ = items.entry(None).or_default().push(item);
+                    }
+                }
                 Value::Catalog(catalog) => {
                     return Err(Error::Backend(format!(
                         "cannot add catalog with id={} to the backend",
                         catalog.id
                     )))
                 }
-                Value::Collection(collection) => self.add_collection(collection).await?,
+                Value::Collection(mut collection) => {
+                    collection.remove_structural_links();
+                    let _ = collection_ids.insert(collection.id.clone());
+                    self.add_collection(collection).await?
+                }
                 Value::ItemCollection(item_collection) => {
-                    items.extend(item_collection.items.into_iter())
+                    for mut item in item_collection.items {
+                        item.remove_structural_links();
+                        if let Some(collection) = item.collection.as_ref() {
+                            let collection = collection.clone();
+                            let _ = item_collection_ids.insert(collection.clone());
+                            let _ = items.entry(Some(collection)).or_default().push(item);
+                        } else {
+                            let _ = items.entry(None).or_default().push(item);
+                        }
+                    }
+                }
+            }
+        }
+        if auto_create_collections {
+            for id in item_collection_ids {
+                if !collection_ids.contains(&id) {
+                    let items = items
+                        .get(&Some(id.clone())) // TODO can we get rid of this clone?
+                        .expect("should have items for collection id");
+                    let collection = Collection::from_id_and_items(id, items);
+                    self.add_collection(collection).await?;
                 }
             }
         }
-        for item in items {
-            self.add_item(item).await?;
+        for (_, items) in items {
+            for item in items {
+                self.add_item(item).await?;
+            }
         }
         Ok(())
     }
@@ -178,3 +222,26 @@ pub trait Backend: Clone + Sync + Send + 'static {
     /// ```
     async fn search(&self, search: Search) -> Result<ItemCollection>;
 }
+
+#[cfg(test)]
+mod tests {
+    use super::Backend;
+    use crate::MemoryBackend;
+
+    #[tokio::test]
+    async fn auto_create_collection() {
+        let mut backend = MemoryBackend::new();
+        backend
+            .add_from_hrefs(
+                &["../spec-examples/v1.0.0/simple-item.json".to_string()],
+                true,
+            )
+            .await
+            .unwrap();
+        let _ = backend
+            .collection("simple-collection")
+            .await
+            .unwrap()
+            .unwrap();
+    }
+}

stac/CHANGELOG.md

@@ -9,10 +9,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Added
 
 - `Bbox` ([#303](https://github.com/stac-utils/stac-rs/pull/303))
+- Functions to create collections from items ([#304](https://github.com/stac-utils/stac-rs/pull/304))
 
 ### Changed
 
-- Use `DateTime<Utc>` instead of `String` for datetimes ([#297](https://github.com/stac-utils/stac-rs/pull/297))
+- Use `DateTime<Utc>` instead of `String` for datetimes ([#297](https://github.com/stac-utils/stac-rs/pull/297), [#304](https://github.com/stac-utils/stac-rs/pull/304))
 - Add `Href.clear_href` ([#299](https://github.com/stac-utils/stac-rs/pull/299))
 
 ### Removed