On-disk storage & single stride size

Cargo.toml

@@ -12,40 +12,49 @@ authors.workspace = true
 rust-version.workspace = true
 license.workspace = true
 
-[workspace]
-members   = ["proc_macros"]
-
 [workspace.package]
-version = "0.4.1"
+version = "0.5.0"
 edition = "2021"
 authors = ["NLnet Labs <[email protected]>"]
 license = "BSD-3-Clause"
-rust-version = "1.80"
+rust-version = "1.82"
 
 [dependencies]
 crossbeam-epoch = "^0.9"
 crossbeam-utils = "^0.8"
+parking_lot_core = "0.9.10"
 
 inetnum   = "0.1"
 log       = "^0.4"
 roaring 	= "0.10.3"
-rotonda-macros = { path = "proc_macros", version = "0.4.0" }
-routecore       = { version = "0.5", features = ["bgp", "bmp", "fsm", "serde"] }
-
+routecore       = { git = "https://github.com/nlnetlabs/routecore", branch = "dev", version = "0.5.2-dev", features = ["bgp", "bmp", "fsm", "serde", "mrt"] }
 ansi_term   = { version = "0.12", optional = true }
 csv         = { version = "1", optional = true }
 rustyline   = { version = "13", optional = true }
-parking_lot_core = "0.9.10"
+clap        = { version = "4.4", optional = true, features = ["derive"] }
+rayon = { version = "1.10", optional = true }
+memmap2 = { version = "0.9", optional = true }
+rand = { version = "0.9" }
+lsm-tree = { version = "2.6.6" }
+serde = "1.0.216"
+serde_derive = "1.0.216"
+serde_json = "1.0.133"
+num-traits = "0.2.19"
+zerocopy = { version = "0.8.17", features = ["derive"] }
 
 [dev-dependencies]
 csv         = { version = "1" }
 env_logger  = { version = "0.10" }
-rand        = "^0.8"
 
 [features]
 cli = ["ansi_term", "rustyline", "csv"]
+mrt = ["clap", "rayon"]
 default = []
 
 [[bin]]
 name = "cli"
 required-features = ["cli"]
+
+[[bin]]
+name = "load_mrt"
+required-features = ["mrt"]

examples/exact_matches.rs

@@ -1,10 +1,19 @@
-use rotonda_store::prelude::*;
-use rotonda_store::prelude::multi::*;
-use rotonda_store::meta_examples::NoMeta;
+use inetnum::addr::Prefix;
+use rotonda_store::match_options::IncludeHistory;
+use rotonda_store::prefix_record::{Record, RouteStatus};
+// use rotonda_store::prelude::multi::*;
+use rotonda_store::{
+    epoch,
+    match_options::{MatchOptions, MatchType},
+    rib::config::MemoryOnlyConfig,
+    rib::StarCastRib,
+    test_types::NoMeta,
+    IntoIpAddr,
+};
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     let guard = &epoch::pin();
-    let tree_bitmap = MultiThreadedStore::<NoMeta>::new()?;
+    let tree_bitmap = StarCastRib::<NoMeta, MemoryOnlyConfig>::try_default()?;
     let pfxs = vec![
         Prefix::new_relaxed(
             0b0000_0000_0000_0000_0000_0000_0000_0000_u32.into_ipaddr(),
@@ -258,7 +267,11 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     for pfx in pfxs.into_iter() {
         println!("insert {}", pfx?);
         // let p : rotonda_store::Prefix<u32, PrefixAs> = pfx.into();
-        tree_bitmap.insert(&pfx.unwrap(), Record::new(0, 0, RouteStatus::Active, NoMeta::Empty), None)?;
+        tree_bitmap.insert(
+            &pfx.unwrap(),
+            Record::new(0, 0, RouteStatus::Active, NoMeta::Empty),
+            None,
+        )?;
     }
     println!("------ end of inserts\n");
     // println!(
@@ -343,9 +356,10 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                 include_withdrawn: false,
                 include_less_specifics: false,
                 include_more_specifics: false,
-                mui: None
+                mui: None,
+                include_history: IncludeHistory::None,
             },
-            guard
+            guard,
         );
         println!("exact match: {:?}", s_spfx);
         println!("-----------");

examples/exact_matches_single.rs

@@ -1,11 +1,16 @@
-use rotonda_store::prelude::*;
-use rotonda_store::SingleThreadedStore;
-use rotonda_store::meta_examples::NoMeta;
+use inetnum::addr::Prefix;
+use rotonda_store::{
+    match_options::{IncludeHistory, MatchOptions, MatchType},
+    prefix_record::{Record, RouteStatus},
+    IntoIpAddr,
+};
+use rotonda_store::{
+    rib::{config::MemoryOnlyConfig, StarCastRib},
+    test_types::NoMeta,
+};
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
-    let v4 = vec![8];
-    let v6 = vec![8];
-    let mut tree_bitmap = SingleThreadedStore::<NoMeta>::new(v4, v6);
+    let tree_bitmap = StarCastRib::<NoMeta, MemoryOnlyConfig>::try_default()?;
 
     let pfxs = vec![
         Prefix::new_relaxed(
@@ -260,7 +265,11 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     for pfx in pfxs.into_iter() {
         println!("insert {}", pfx?);
         // let p : rotonda_store::Prefix<u32, PrefixAs> = pfx.into();
-        tree_bitmap.insert(&pfx.unwrap(), NoMeta::Empty)?;
+        tree_bitmap.insert(
+            &pfx.unwrap(),
+            Record::new(1, 0, RouteStatus::Active, NoMeta::Empty),
+            None,
+        )?;
     }
     println!("------ end of inserts\n");
     // println!(
@@ -336,7 +345,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         // Prefix::new_relaxed(std::net::Ipv4Addr::new(1, 0, 128, 0).into(), 24),
     ] {
         println!("search for: {:?}", spfx);
-        // let locks = tree_bitmap.acquire_prefixes_rwlock_read();
+        let guard = &rotonda_store::epoch::pin();
         let s_spfx = tree_bitmap.match_prefix(
             // (&locks.0, &locks.1),
             &spfx.unwrap(),
@@ -345,8 +354,10 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                 include_withdrawn: false,
                 include_less_specifics: false,
                 include_more_specifics: false,
-                mui: None
+                mui: None,
+                include_history: IncludeHistory::None,
             },
+            guard,
         );
         println!("exact match: {:?}", s_spfx);
         println!("-----------");

examples/full_table_multiple_trees_json.rs

@@ -1,17 +1,20 @@
-// extern crate self as roto;
-use rotonda_store::prelude::*;
-use rotonda_store::prelude::multi::*;
-use rotonda_store::meta_examples::PrefixAs;
+use inetnum::addr::Prefix;
+use rotonda_store::epoch;
+use rotonda_store::match_options::{IncludeHistory, MatchOptions, MatchType};
+use rotonda_store::prefix_record::{PrefixRecord, Record, RouteStatus};
+use rotonda_store::rib::config::MemoryOnlyConfig;
+use rotonda_store::rib::StarCastRib;
+use rotonda_store::test_types::PrefixAs;
 
 use std::error::Error;
 use std::fs::File;
 use std::process;
 
-#[create_store((
-    [4, 4, 4, 4, 4, 4, 4, 4],
-    [3,4,5,4]
-))]
-struct MyStore;
+// #[create_store((
+//     ([4, 4, 4, 4, 4, 4, 4, 4], 5, 17),
+//     ([3, 4, 5, 4], 17, 29)
+// ))]
+// struct MyStore;
 
 fn main() -> Result<(), Box<dyn Error>> {
     const CSV_FILE_PATH: &str = "./data/uniq_pfx_asn_dfz_rnd.csv";
@@ -32,7 +35,12 @@ fn main() -> Result<(), Box<dyn Error>> {
             let asn: u32 = record[2].parse().unwrap();
             let pfx = PrefixRecord::<PrefixAs>::new(
                 Prefix::new(net.into(), len)?,
-                vec![Record::new(0, 0, RouteStatus::Active, PrefixAs(asn))],
+                vec![Record::new(
+                    0,
+                    0,
+                    RouteStatus::Active,
+                    PrefixAs::new(asn.into()),
+                )],
             );
             pfxs.push(pfx);
         }
@@ -46,7 +54,9 @@ fn main() -> Result<(), Box<dyn Error>> {
         println!("[");
         for n in 1..6 {
             let mut rec_vec: Vec<PrefixRecord<PrefixAs>> = vec![];
-            let tree_bitmap = MyStore::<PrefixAs>::new()?;
+            let config = MemoryOnlyConfig;
+            let tree_bitmap =
+                StarCastRib::<PrefixAs, _>::new_with_config(config)?;
 
             if let Err(err) = load_prefixes(&mut rec_vec) {
                 println!("error running example: {}", err);
@@ -85,10 +95,11 @@ fn main() -> Result<(), Box<dyn Error>> {
                                     include_withdrawn: false,
                                     include_less_specifics: false,
                                     include_more_specifics: false,
-                                    mui: None
+                                    mui: None,
+                                    include_history: IncludeHistory::None,
                                 },
-                                guard
-                            );
+                                guard,
+                            )?;
                         }
                     }
                 }
@@ -101,26 +112,24 @@ fn main() -> Result<(), Box<dyn Error>> {
 
             println!("{{");
             println!("\"type\": \"treebitmap_univec\",");
-            println!(
-                "\"strides v4 \": {:?},",
-                &tree_bitmap
-                    .v4
-                    .store
-                    .get_stride_sizes()
-                    .iter()
-                    .map_while(|s| if s > &0 { Some(*s) } else { None })
-                    .collect::<Vec<_>>()
-            );
-            println!(
-                "\"strides v6 \": {:?},",
-                &tree_bitmap
-                    .v6
-                    .store
-                    .get_stride_sizes()
-                    .iter()
-                    .map_while(|s| if s > &0 { Some(*s) } else { None })
-                    .collect::<Vec<_>>()
-            );
+            // println!(
+            //     "\"strides v4 \": {:?},",
+            //     &tree_bitmap
+            //         .v4
+            //         .get_stride_sizes()
+            //         .iter()
+            //         .map_while(|s| if s > &0 { Some(*s) } else { None })
+            //         .collect::<Vec<_>>()
+            // );
+            // println!(
+            //     "\"strides v6 \": {:?},",
+            //     &tree_bitmap
+            //         .v6
+            //         .get_stride_sizes()
+            //         .iter()
+            //         .map_while(|s| if s > &0 { Some(*s) } else { None })
+            //         .collect::<Vec<_>>()
+            // );
             println!("\"run_no\": {},", n);
             println!("\"inserts_num\": {},", inserts_num);
             println!("\"insert_duration_nanos\": {},", dur_insert_nanos);

examples/more_specifics.rs

@@ -1,14 +1,14 @@
-use rotonda_store::meta_examples::PrefixAs;
-use rotonda_store::prelude::*;
-use rotonda_store::prelude::multi::*;
-
-use rotonda_store::AddressFamily;
 use inetnum::addr::Prefix;
+use rotonda_store::{
+    epoch,
+    match_options::{IncludeHistory, MatchOptions, MatchType},
+    prefix_record::{Record, RouteStatus},
+    rib::{config::MemoryOnlyConfig, StarCastRib},
+    test_types::PrefixAs,
+    IntoIpAddr,
+};
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
-    // type StoreType = InMemStorage<u32, PrefixAs>;
-    let tree_bitmap =
-        MultiThreadedStore::<PrefixAs>::new()?;
     let pfxs = vec![
         Prefix::new_relaxed(
             0b0000_0000_0000_0000_0000_0000_0000_0000_u32.into_ipaddr(),
@@ -215,7 +215,11 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     for pfx in pfxs.into_iter() {
         // println!("insert {:?}", pfx);
         let p: Prefix = pfx.unwrap();
-        tree_bitmap.insert(&p, Record::new(0,0, RouteStatus::Active, PrefixAs(666)), None)?;
+        StarCastRib::<PrefixAs, MemoryOnlyConfig>::try_default()?.insert(
+            &p,
+            Record::new(0, 0, RouteStatus::Active, PrefixAs::new(666.into())),
+            None,
+        )?;
     }
     println!("------ end of inserts\n");
     // println!(
@@ -277,17 +281,20 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     ] {
         println!("search for: {:?}", spfx);
         let guard = &epoch::pin();
-        let s_spfx = tree_bitmap.match_prefix(
-            &spfx.unwrap(),
-            &MatchOptions {
-                match_type: MatchType::ExactMatch,
-                include_withdrawn: false,
-                include_less_specifics: true,
-                include_more_specifics: true,
-                mui: None
-            },
-            guard
-        );
+        let s_spfx =
+            StarCastRib::<PrefixAs, MemoryOnlyConfig>::try_default()?
+                .match_prefix(
+                    &spfx.unwrap(),
+                    &MatchOptions {
+                        match_type: MatchType::ExactMatch,
+                        include_withdrawn: false,
+                        include_less_specifics: true,
+                        include_more_specifics: true,
+                        mui: None,
+                        include_history: IncludeHistory::None,
+                    },
+                    guard,
+                );
         println!("em/m-s: {:#?}", s_spfx);
         println!("-----------");
     }